Example Notebook

[1]:

%matplotlib inline
%load_ext autoreload
%autoreload 2

[2]:

import sparkmon

[3]:

from pyspark.sql import SparkSession
spark = (
    SparkSession.builder.appName("Python Spark SQL basic example")
    .config("spark.some.config.option", "some-value")
    .getOrCreate()
)

Standard usage

[4]:

# Advance usage for remote usage:
# app = sparkmon.create_application_from_link()
# app = sparkmon.create_application_from_spark(spark)
# mon = sparkmon.SparkMon(app, period=5)

[7]:

# Simple usage:
mon = sparkmon.SparkMon(spark, period=5)
mon.start()

app = mon.application

[8]:

# This cannot be used with callbacks at the moment
mon.live_plot_notebook(n_iter=2)

Development

[9]:

# Re trigger parsing for development purposes
app.debug = True
app.parse_db()

[10]:

mon.application.plot()

[11]:

# mon.application.timeseries_db

[12]:

# mon.stop()

[13]:

# spark.stop()

[ ]:

[ ]: