Heat Consumption AD problem with TIM Engine API Client for Python

1. Set up Python Libraries

import logging
import pandas as pd
import plotly as plt
import plotly.express as px
import plotly.graph_objects as go
import numpy as np
import json
import copy

import tim_client

2. Credentials and logging

(Do not forget to fill in your credentials in the credentials.json file)

with open('credentials.json') as f:
    credentials_json = json.load(f)

TIM_URL = 'https://timws.tangent.works/v4/api'

SAVE_JSON = False
JSON_SAVING_FOLDER = 'logs/'

LOGGING_LEVEL = 'INFO'

level = logging.getLevelName(LOGGING_LEVEL)
logging.basicConfig(level=level, format='[%(levelname)s] %(asctime)s - %(name)s:%(funcName)s:%(lineno)s - %(message)s')
logger = logging.getLogger(__name__)

credentials = tim_client.Credentials(credentials_json['license_key'], credentials_json['email'], credentials_json['password'], tim_url=TIM_URL)
api_client = tim_client.ApiClient(credentials)

api_client.save_json = SAVE_JSON
api_client.json_saving_folder_path = JSON_SAVING_FOLDER

[INFO] 2020-11-03 12:33:56,954 - tim_client.api_client:save_json:66 - Saving JSONs functionality has been disabled
[INFO] 2020-11-03 12:33:56,958 - tim_client.api_client:json_saving_folder_path:75 - JSON destination folder changed to logs

3. Data preparation

3.1 Load Data

data = tim_client.load_dataset_from_csv_file('HeatConsumption.csv', sep=';')
nrow, ncol = data.shape

print("Number of rows: ", nrow)
print("Number of columns: ", ncol)

Number of rows:  439
Number of columns:  8

3.2 Change row indices to datetimes

(only for visualization purposes, not required for calling the tim_client methods)

data.index = pd.to_datetime(data['Date'])

3.3 Visualize Data

cols = data.columns

fig = plt.subplots.make_subplots(rows=2, cols=1, shared_xaxes=True, vertical_spacing=0.02)
fig.add_trace(go.Scatter(x=data.loc[:, "Date"], y=data.loc[:, "HeatConsumption"], name="HeatConsumption"), row=1, col=1)
fig.add_trace(go.Scatter(x=data.loc[:, "Date"], y=data.loc[:, "FlowTemperature"], name="FlowTemperature"), row=2, col=1)
fig.add_trace(go.Scatter(x=data.loc[:, "Date"], y=data.loc[:, "ReturnTemperature"], name="ReturnTemperature"), row=2, col=1)
fig.add_trace(go.Scatter(x=data.loc[:, "Date"], y=data.loc[:, "VolumeFlow"], name="VolumeFlow"), row=2, col=1)
fig.add_trace(go.Scatter(x=data.loc[:, "Date"], y=data.loc[:, "OutsideTemperature"], name="OutsideTemperature"), row=2, col=1)
fig.add_trace(go.Scatter(x=data.loc[:, "Date"], y=data.loc[:, "WindSpeed"], name="WindSpeed"), row=2, col=1)
fig.add_trace(go.Scatter(x=data.loc[:, "Date"], y=data.loc[:, "WindDirection"], name="WindDirection"), row=2, col=1)

fig.update_layout(height=700, width=1000, title_text="Data visualization")

fig.show()

4. Model Building

4.1 Configuration/TIM Setup

configuration = {
    "sensitivity": 1.5,
    "normalBehaviorModellingConfiguration": {
      "useTargetOffsets": False
    },
    "abnormalBehaviorModellingConfiguration": {
      "features": [
        "Residual",
      ]
    },
    "extendedOutputConfiguration": {
      "returnAnomalyIndicator": True,
      "returnSimpleImportances": True,
      "returnExtendedImportances": True
    }
}

4.2 Select data

train_until = "2018-10-25 00:00:00"
data_train = data[:train_until]

4.3 Build a model

model = api_client.detection_build_model(data_train, configuration)
model.status

'FinishedWithWarning'

4. Visualize Results

4.1 In-sample anomaly detection

normal_behavior = model.normal_behavior
anomaly_indicator = model.anomaly_indicator
anomaly_indices = [i for i, ai in enumerate(anomaly_indicator.loc[:, "Anomaly Indicator"]) if ai > 1]

fig1 = plt.subplots.make_subplots(rows=2, cols=1, shared_xaxes=True, vertical_spacing=0.02)
fig1.add_trace(go.Scatter(x=data.loc[:, "Date"], y=data.loc[:, "HeatConsumption"], name="HeatConsumption", 
                          line=dict(color="black")), row=1, col=1)
    
fig1.add_trace(go.Scatter(x=[data.iloc[0, 0], data.iloc[len(data) - 1, 0]], y=[1,1], name="threshold", 
                          line=dict(color="red"), mode="lines"), row=2, col=1)
    
# in-sample anomalies, anomaly indicators and normal behavior

fig1.add_trace(go.Scatter(x=normal_behavior.index, y=normal_behavior["Normal Behavior"], name="Normal Behavior", 
                        line=dict(color="darkgoldenrod")), row=1, col=1)

fig1.add_trace(go.Scatter(x=anomaly_indicator.index, y=anomaly_indicator["Anomaly Indicator"], name="Anomaly Indicator", 
                        line=dict(color="darkgrey")), row=2, col=1)

anomalies = anomaly_indicator[anomaly_indicator["Anomaly Indicator"] > 1]

fig1.add_trace(go.Scatter(x=data.loc[anomalies.index.values, "Date"], y=data.loc[anomalies.index.values, "HeatConsumption"], mode='markers', 
                        marker=dict(color="red", size=6), name="Anomalies"), row=1, col=1)
    
fig1.update_layout(height=700, width=1000, title_text="In-sample anomaly detection")

fig1.show()

4.2 Retrieve influencers and term importances

si_df = pd.DataFrame(model.predictors_importances['simpleImportances'])
si_df.sort_values(by=['importance'], ascending=False)
ei_df = pd.DataFrame(model.predictors_importances['extendedImportances'])
ei_df = ei_df.sort_values(by=['time', 'importance'], ascending=False)

4.3 Visualize influencers(predictors) importances

fig2 = go.Figure(go.Bar(x=si_df['predictorName'], y=si_df['importance']))
fig2.update_layout(height=400,
                  width=600, 
                  title_text="Importances of predictors",
                  xaxis_title="Predictor name",
                  yaxis_title="Predictor importance (%)")
fig2.show()

4.4 Visualize term(features) importances

fig3 = go.Figure(go.Bar(x=ei_df['termName'], y=ei_df['importance']))
fig3.update_layout(height=700,
                  width=1000,
                  title_text="Importances of features",
                  xaxis_title="Feature name",
                  yaxis_title="Feature importance (%)")
fig3.show()