From d3a438629139554bb709c911c95754663cb249e7 Mon Sep 17 00:00:00 2001 From: xeniape Date: Thu, 5 Mar 2026 11:10:57 +0100 Subject: [PATCH 01/22] end-to-end-security demo in non-default ns and with demo labels --- demos/end-to-end-security/create-spark-report.yaml | 7 +++++-- demos/end-to-end-security/serviceaccount.yaml | 2 -- stacks/_templates/keycloak-serviceaccount.yaml | 2 -- stacks/_templates/keycloak.yaml | 7 ++++++- stacks/_templates/postgresql-hive-iceberg.yaml | 1 + stacks/_templates/postgresql-superset.yaml | 1 + stacks/end-to-end-security/hdfs-regorules.yaml | 10 +++++----- stacks/end-to-end-security/kerberos-secretclass.yaml | 6 +++--- stacks/end-to-end-security/krb5.yaml | 3 +++ stacks/end-to-end-security/opa.yaml | 2 +- stacks/end-to-end-security/trino.yaml | 6 +++--- stacks/stacks-v2.yaml | 5 ++++- 12 files changed, 32 insertions(+), 20 deletions(-) diff --git a/demos/end-to-end-security/create-spark-report.yaml b/demos/end-to-end-security/create-spark-report.yaml index ff48f9d9..5c943eb2 100644 --- a/demos/end-to-end-security/create-spark-report.yaml +++ b/demos/end-to-end-security/create-spark-report.yaml @@ -53,6 +53,9 @@ data: kind: SparkApplication metadata: name: spark-report + labels: + stackable.tech/vendor: Stackable + stackable.tech/demo: end-to-end-security spec: sparkImage: productVersion: 4.0.1 @@ -64,10 +67,10 @@ data: sparkConf: spark.driver.extraClassPath: /stackable/config/hdfs spark.executor.extraClassPath: /stackable/config/hdfs - spark.hadoop.hive.metastore.kerberos.principal: hive/hive-iceberg.default.svc.cluster.local@KNAB.COM + spark.hadoop.hive.metastore.kerberos.principal: hive/hive-iceberg.{{ NAMESPACE }}.svc.cluster.local@KNAB.COM spark.hadoop.hive.metastore.sasl.enabled: "true" spark.kerberos.keytab: /stackable/kerberos/keytab - spark.kerberos.principal: spark/spark.default.svc.cluster.local@KNAB.COM + spark.kerberos.principal: spark/spark.{{ NAMESPACE }}.svc.cluster.local@KNAB.COM spark.sql.catalog.lakehouse: org.apache.iceberg.spark.SparkCatalog spark.sql.catalog.lakehouse.type: hive spark.sql.catalog.lakehouse.uri: thrift://hive-iceberg-metastore:9083 diff --git a/demos/end-to-end-security/serviceaccount.yaml b/demos/end-to-end-security/serviceaccount.yaml index cb3ea626..df7d2cf0 100644 --- a/demos/end-to-end-security/serviceaccount.yaml +++ b/demos/end-to-end-security/serviceaccount.yaml @@ -3,7 +3,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: demo-serviceaccount - namespace: default --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -12,7 +11,6 @@ metadata: subjects: - kind: ServiceAccount name: demo-serviceaccount - namespace: default roleRef: kind: ClusterRole name: demo-clusterrole diff --git a/stacks/_templates/keycloak-serviceaccount.yaml b/stacks/_templates/keycloak-serviceaccount.yaml index df485e5e..16ac31d8 100644 --- a/stacks/_templates/keycloak-serviceaccount.yaml +++ b/stacks/_templates/keycloak-serviceaccount.yaml @@ -3,7 +3,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: keycloak-serviceaccount - namespace: default --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -12,7 +11,6 @@ metadata: subjects: - kind: ServiceAccount name: keycloak-serviceaccount - namespace: default roleRef: kind: ClusterRole name: keycloak-clusterrole diff --git a/stacks/_templates/keycloak.yaml b/stacks/_templates/keycloak.yaml index 0ce30d4e..2983f5ab 100644 --- a/stacks/_templates/keycloak.yaml +++ b/stacks/_templates/keycloak.yaml @@ -59,18 +59,23 @@ spec: while : do echo "Determining Keycloak public reachable address" - KEYCLOAK_ADDRESS=$(kubectl get svc keycloak -o json | jq -r --slurpfile endpoints <(kubectl get endpoints keycloak -o json) --slurpfile nodes <(kubectl get nodes -o json) '($nodes[0].items[] | select(.metadata.name == $endpoints[0].subsets[].addresses[].nodeName) | .status.addresses | map(select(.type == "ExternalIP" or .type == "InternalIP")) | min_by(.type) | .address | tostring) + ":" + (.spec.ports[] | select(.name == "https") | .nodePort | tostring)') + KEYCLOAK_ADDRESS=$(kubectl get svc keycloak -n {{ NAMESPACE }} -o json | jq -r --slurpfile endpoints <(kubectl get endpoints keycloak -n {{ NAMESPACE }} -o json) --slurpfile nodes <(kubectl get nodes -o json) '($nodes[0].items[] | select(.metadata.name == $endpoints[0].subsets[].addresses[].nodeName) | .status.addresses | map(select(.type == "ExternalIP" or .type == "InternalIP")) | min_by(.type) | .address | tostring) + ":" + (.spec.ports[] | select(.name == "https") | .nodePort | tostring)') echo "Found Keycloak running at $KEYCLOAK_ADDRESS" if [ ! -z "$KEYCLOAK_ADDRESS" ]; then KEYCLOAK_HOSTNAME="$(echo $KEYCLOAK_ADDRESS | grep -oP '^[^:]+')" KEYCLOAK_PORT="$(echo $KEYCLOAK_ADDRESS | grep -oP '[0-9]+$')" + STACK_LABEL=$(kubectl get svc keycloak -n {{ NAMESPACE }} -o json | jq '.metadata.labels."stackable.tech/stack"') + cat << EOF | kubectl apply -f - apiVersion: authentication.stackable.tech/v1alpha1 kind: AuthenticationClass metadata: name: keycloak + labels: + stackable.tech/vendor: Stackable + stackable.tech/stack: $STACK_LABEL spec: provider: oidc: diff --git a/stacks/_templates/postgresql-hive-iceberg.yaml b/stacks/_templates/postgresql-hive-iceberg.yaml index 7756d656..ba9ddbd3 100644 --- a/stacks/_templates/postgresql-hive-iceberg.yaml +++ b/stacks/_templates/postgresql-hive-iceberg.yaml @@ -8,6 +8,7 @@ version: 18.1.3 # 18.0.0 options: commonLabels: stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} auth: username: hive password: hive diff --git a/stacks/_templates/postgresql-superset.yaml b/stacks/_templates/postgresql-superset.yaml index 0733d1d9..cbb3af09 100644 --- a/stacks/_templates/postgresql-superset.yaml +++ b/stacks/_templates/postgresql-superset.yaml @@ -8,6 +8,7 @@ version: 18.1.3 # 18.0.0 options: commonLabels: stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} auth: username: superset password: superset diff --git a/stacks/end-to-end-security/hdfs-regorules.yaml b/stacks/end-to-end-security/hdfs-regorules.yaml index 9736e6d6..f45f4230 100644 --- a/stacks/end-to-end-security/hdfs-regorules.yaml +++ b/stacks/end-to-end-security/hdfs-regorules.yaml @@ -211,21 +211,21 @@ data: } groups_for_user := { - "trino/trino.default.svc.cluster.local@KNAB.COM": [], - "spark/spark.default.svc.cluster.local@KNAB.COM": [], - "hive/hive.default.svc.cluster.local@KNAB.COM": [], + "trino/trino.{{ NAMESPACE }}.svc.cluster.local@KNAB.COM": [], + "spark/spark.{{ NAMESPACE }}.svc.cluster.local@KNAB.COM": [], + "hive/hive.{{ NAMESPACE }}.svc.cluster.local@KNAB.COM": [], } groups_for_short_user_name := {} acls := [ { - "identity": "user:trino/trino.default.svc.cluster.local@KNAB.COM", + "identity": "user:trino/trino.{{ NAMESPACE }}.svc.cluster.local@KNAB.COM", "action": "rw", "resource": "hdfs:dir:/lakehouse/", }, { - "identity": "user:spark/spark.default.svc.cluster.local@KNAB.COM", + "identity": "user:spark/spark.{{ NAMESPACE }}.svc.cluster.local@KNAB.COM", "action": "rw", "resource": "hdfs:dir:/lakehouse/", }, diff --git a/stacks/end-to-end-security/kerberos-secretclass.yaml b/stacks/end-to-end-security/kerberos-secretclass.yaml index 51a98379..cf54febf 100644 --- a/stacks/end-to-end-security/kerberos-secretclass.yaml +++ b/stacks/end-to-end-security/kerberos-secretclass.yaml @@ -7,12 +7,12 @@ spec: backend: kerberosKeytab: realmName: KNAB.COM - kdc: krb5-kdc.default.svc.cluster.local + kdc: krb5-kdc.{{ NAMESPACE }}.svc.cluster.local admin: mit: - kadminServer: krb5-kdc.default.svc.cluster.local + kadminServer: krb5-kdc.{{ NAMESPACE }}.svc.cluster.local adminKeytabSecret: - namespace: default + namespace: {{ NAMESPACE }} name: secret-operator-keytab adminPrincipal: stackable-secret-operator --- diff --git a/stacks/end-to-end-security/krb5.yaml b/stacks/end-to-end-security/krb5.yaml index 06b31066..bb45e4e6 100644 --- a/stacks/end-to-end-security/krb5.yaml +++ b/stacks/end-to-end-security/krb5.yaml @@ -77,6 +77,9 @@ spec: volumeClaimTemplates: - metadata: name: data + labels: + stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} spec: accessModes: - ReadWriteOnce diff --git a/stacks/end-to-end-security/opa.yaml b/stacks/end-to-end-security/opa.yaml index bcf3bcad..61a41906 100644 --- a/stacks/end-to-end-security/opa.yaml +++ b/stacks/end-to-end-security/opa.yaml @@ -10,7 +10,7 @@ spec: userInfo: backend: keycloak: - hostname: keycloak.default.svc.cluster.local + hostname: keycloak.{{ NAMESPACE }}.svc.cluster.local port: 8443 tls: verification: diff --git a/stacks/end-to-end-security/trino.yaml b/stacks/end-to-end-security/trino.yaml index e2ced864..555cc5c9 100644 --- a/stacks/end-to-end-security/trino.yaml +++ b/stacks/end-to-end-security/trino.yaml @@ -83,15 +83,15 @@ spec: configOverrides: # HDFS configuration hive.hdfs.authentication.type: KERBEROS - hive.hdfs.trino.principal: trino/trino.default.svc.cluster.local@KNAB.COM + hive.hdfs.trino.principal: trino/trino.{{ NAMESPACE }}.svc.cluster.local@KNAB.COM hive.hdfs.trino.keytab: /stackable/kerberos/keytab hive.hdfs.impersonation.enabled: "false" hive.hdfs.wire-encryption.enabled: "true" # HMS configuration hive.metastore.authentication.type: KERBEROS - hive.metastore.client.principal: trino/trino.default.svc.cluster.local@KNAB.COM + hive.metastore.client.principal: trino/trino.{{ NAMESPACE }}.svc.cluster.local@KNAB.COM hive.metastore.client.keytab: /stackable/kerberos/keytab - hive.metastore.service.principal: hive/hive-iceberg.default.svc.cluster.local@KNAB.COM + hive.metastore.service.principal: hive/hive-iceberg.{{ NAMESPACE }}.svc.cluster.local@KNAB.COM hive.metastore.thrift.impersonation.enabled: "false" # By default, Hive views are executed with the RUN AS DEFINER security mode. Set the hive.hive-views.run-as-invoker catalog configuration property to true to use RUN AS INVOKER semantics. # However, this does *not* work for Iceberg catalogs :/ (I asked on the Trino slack: https://trinodb.slack.com/archives/CJ6UC075E/p1711449384648869) diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index 25ba0cc9..a4466bcc 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -571,7 +571,7 @@ stacks: - superset - opa - keycloak - supportedNamespaces: ["default"] # SecretClass "kerberos" needs explicit namespace in adminKeytabSecret + supportedNamespaces: [] resourceRequests: cpu: 7000m memory: 19586Mi @@ -597,6 +597,9 @@ stacks: - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/setup-postgresql.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/superset.yaml parameters: + - name: stackLabel + description: Stack label to add to various components + default: end-to-end-security - name: keycloakAdminPassword description: Password of the Keycloak admin user default: adminadmin From 65d4049beec11a232f958c48a869f143ef6d62eb Mon Sep 17 00:00:00 2001 From: xeniape Date: Thu, 5 Mar 2026 21:19:27 +0100 Subject: [PATCH 02/22] airflow-scheduled-job demo in non-default ns and with demo labels --- demos/airflow-scheduled-job/create-trino-tables.yaml | 2 +- demos/airflow-scheduled-job/serviceaccount.yaml | 3 +-- stacks/_templates/postgresql-airflow.yaml | 1 + stacks/airflow/airflow.yaml | 2 +- stacks/airflow/rbac.yaml | 3 --- stacks/airflow/s3-connection.yaml | 2 +- stacks/airflow/trino.yaml | 4 ++-- stacks/stacks-v2.yaml | 7 +++++-- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/demos/airflow-scheduled-job/create-trino-tables.yaml b/demos/airflow-scheduled-job/create-trino-tables.yaml index 175915bc..802de748 100644 --- a/demos/airflow-scheduled-job/create-trino-tables.yaml +++ b/demos/airflow-scheduled-job/create-trino-tables.yaml @@ -9,7 +9,7 @@ spec: serviceAccountName: demo-serviceaccount containers: - name: create-tables-in-trino - image: oci.stackable.tech/sdp/testing-tools:0.3.0-stackable0.0.0-dev + image: oci.stackable.tech/sdp/testing-tools/trino:0.3.0-stackable0.0.0-dev command: ["bash", "-c", "python -u /tmp/script/script.py"] volumeMounts: - name: script diff --git a/demos/airflow-scheduled-job/serviceaccount.yaml b/demos/airflow-scheduled-job/serviceaccount.yaml index 90f2f9b2..e42b2e26 100644 --- a/demos/airflow-scheduled-job/serviceaccount.yaml +++ b/demos/airflow-scheduled-job/serviceaccount.yaml @@ -3,7 +3,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: demo-serviceaccount - namespace: default --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -12,7 +11,7 @@ metadata: subjects: - kind: ServiceAccount name: demo-serviceaccount - namespace: default + namespace: {{ NAMESPACE }} roleRef: kind: ClusterRole name: demo-clusterrole diff --git a/stacks/_templates/postgresql-airflow.yaml b/stacks/_templates/postgresql-airflow.yaml index 08d1e232..dda36a74 100644 --- a/stacks/_templates/postgresql-airflow.yaml +++ b/stacks/_templates/postgresql-airflow.yaml @@ -8,6 +8,7 @@ version: 18.1.3 # 18.0.0 options: commonLabels: stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} auth: username: airflow password: airflow diff --git a/stacks/airflow/airflow.yaml b/stacks/airflow/airflow.yaml index ba514959..ea9ff560 100644 --- a/stacks/airflow/airflow.yaml +++ b/stacks/airflow/airflow.yaml @@ -198,7 +198,7 @@ data: ) ), V1EnvVar(name="TRINO_USER", value="admin"), - V1EnvVar(name="TRINO_HOST", value="trino-coordinator-default-headless.default.svc.cluster.local"), + V1EnvVar(name="TRINO_HOST", value="trino-coordinator-default-headless.{{ NAMESPACE }}.svc.cluster.local"), V1EnvVar(name="TRINO_PORT", value="8443"), V1EnvVar(name="CERT_PATH", value="/dbt/trusted/ca.crt"), ], diff --git a/stacks/airflow/rbac.yaml b/stacks/airflow/rbac.yaml index 8c27213e..fd599d6e 100644 --- a/stacks/airflow/rbac.yaml +++ b/stacks/airflow/rbac.yaml @@ -2,7 +2,6 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - namespace: default name: airflow-events-reader rules: - apiGroups: [""] @@ -13,11 +12,9 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: airflow-events-binding - namespace: default subjects: - kind: ServiceAccount name: airflow-serviceaccount - namespace: default roleRef: kind: Role name: airflow-events-reader diff --git a/stacks/airflow/s3-connection.yaml b/stacks/airflow/s3-connection.yaml index 411d7a4d..610e6541 100644 --- a/stacks/airflow/s3-connection.yaml +++ b/stacks/airflow/s3-connection.yaml @@ -4,7 +4,7 @@ kind: S3Connection metadata: name: minio spec: - host: minio.default.svc.cluster.local + host: minio.{{ NAMESPACE }}.svc.cluster.local port: 9000 accessStyle: Path credentials: diff --git a/stacks/airflow/trino.yaml b/stacks/airflow/trino.yaml index 0fa4244d..60c2189a 100644 --- a/stacks/airflow/trino.yaml +++ b/stacks/airflow/trino.yaml @@ -41,7 +41,7 @@ spec: ca: secret: name: secret-provisioner-trino-tls-ca - namespace: default + namespace: {{ NAMESPACE }} autoGenerate: true --- apiVersion: secrets.stackable.tech/v1alpha1 @@ -54,7 +54,7 @@ spec: ca: secret: name: secret-provisioner-trino-internal-tls-ca - namespace: default + namespace: {{ NAMESPACE }} autoGenerate: true --- apiVersion: authentication.stackable.tech/v1alpha1 diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index a4466bcc..69aefb65 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -155,7 +155,7 @@ stacks: - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-airflow.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-hive-iceberg.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/airflow/kafka.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/trino-iceberg/s3-connection.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/airflow/s3-connection.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/airflow/hive-metastores.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/airflow/trino.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/airflow/airflow.yaml @@ -163,12 +163,15 @@ stacks: - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/airflow/opa-rules.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/airflow/opa.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/airflow/minio.yaml - supportedNamespaces: ["default"] # auto-generated secret classes need to be templated + supportedNamespaces: [] resourceRequests: cpu: 3400m memory: 9010Mi pvc: 24Gi parameters: + - name: stackLabel + description: Stack label to add to various components + default: airflow - name: trinoAdminPassword description: Password of the Trino admin user default: adminadmin From 3ad6451804376fe09fbbc1e1897aa8906170c38b Mon Sep 17 00:00:00 2001 From: xeniape Date: Mon, 9 Mar 2026 08:54:30 +0100 Subject: [PATCH 03/22] nifi-kafka-druid-earthquake-data demo in non-default ns and with demo labels --- demos/demos-v2.yaml | 2 +- .../create-druid-ingestion-job.yaml | 2 +- .../create-nifi-ingestion-job.yaml | 3 +-- demos/nifi-kafka-druid-earthquake-data/serviceaccount.yaml | 3 +-- stacks/_templates/postgresql-druid.yaml | 1 + 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/demos/demos-v2.yaml b/demos/demos-v2.yaml index eb86a12d..5e481664 100644 --- a/demos/demos-v2.yaml +++ b/demos/demos-v2.yaml @@ -111,7 +111,7 @@ demos: - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml - supportedNamespaces: ["default"] + supportedNamespaces: [] resourceRequests: cpu: 8700m memory: 42034Mi diff --git a/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml b/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml index 50aae834..5ede83cd 100644 --- a/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml +++ b/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml @@ -52,7 +52,7 @@ data: "ioConfig": { "type": "kafka", "consumerProperties": { - "bootstrap.servers": "kafka-broker-default-headless.default.svc.cluster.local:9093", + "bootstrap.servers": "kafka-broker-default-headless.{{ NAMESPACE }}.svc.cluster.local:9093", "security.protocol": "SSL", "ssl.truststore.location": "/stackable/tls/truststore.p12", "ssl.truststore.password": "changeit", diff --git a/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml b/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml index b86ea300..2b35738f 100644 --- a/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml +++ b/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml @@ -127,7 +127,6 @@ apiVersion: v1 kind: ConfigMap metadata: name: ingest-earthquake-data-flow - namespace: default data: IngestEarthquakesToKafka.json: | - { "flowContents": { "identifier": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4", "instanceIdentifier": "a52fc536-0195-1000-ffff-ffffb77a46e7", "name": "IngestEarthquakesToKafka_raw", "comments": "", "position": { "x": -408.0, "y": -176.0 }, "processGroups": [], "remoteProcessGroups": [], "processors": [ { "identifier": "6f74b992-9f25-3fa8-ada3-eebe335159b3", "instanceIdentifier": "a5301fb3-0195-1000-0000-0000734e0ae7", "name": "SplitRecord", "comments": "", "position": { "x": 472.0, "y": 376.0 }, "type": "org.apache.nifi.processors.standard.SplitRecord", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Record Writer": "5e81a80d-728d-3c87-800e-aa957618da55", "Record Reader": "d082021f-5f72-322a-8e1d-c5990d04b2b1", "Records Per Split": "2000" }, "propertyDescriptors": { "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Records Per Split": { "name": "Records Per Split", "displayName": "Records Per Split", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "original", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4" }, { "identifier": "e29d8c86-db41-3f3e-9ba7-bffeaefcade6", "instanceIdentifier": "d86d6b2e-ed8f-3d61-9200-5c02c0918eec", "name": "InvokeHTTP", "comments": "", "position": { "x": 473.00000076287904, "y": 142.99998844850802 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://repo.stackable.tech/repository/misc/earthquake-data/earthquakes_1950_to_2022.csv", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "true", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "365000 days", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "PRIMARY", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4" }, { "identifier": "cd097bd8-d55f-3685-b63f-56d46d14c251", "instanceIdentifier": "2ab64ebf-3492-348b-8b9a-7a97e920220f", "name": "PublishKafka", "comments": "", "position": { "x": 472.0, "y": 608.0 }, "type": "org.apache.nifi.kafka.processors.PublishKafka", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-nar", "version": "2.6.0" }, "properties": { "compression.type": "none", "acks": "all", "Message Demarcator": null, "Kafka Key Attribute Encoding": "utf-8", "Record Reader": "d082021f-5f72-322a-8e1d-c5990d04b2b1", "Record Metadata Strategy": "FROM_PROPERTIES", "Transactional ID Prefix": null, "Header Encoding": "UTF-8", "max.request.size": "1 MB", "Topic Name": "earthquakes", "Kafka Key": null, "partition": null, "Kafka Connection Service": "db258647-11aa-30c5-a8c6-a49a4296cf19", "Publish Strategy": "USE_VALUE", "Record Key Writer": null, "Failure Strategy": "Route to Failure", "partitioner.class": "org.apache.kafka.clients.producer.internals.DefaultPartitioner", "Record Writer": "f3254a5e-0024-31c3-b59a-9edff2a099c9", "Transactions Enabled": "false", "Message Key Field": null, "FlowFile Attribute Header Pattern": null }, "propertyDescriptors": { "compression.type": { "name": "compression.type", "displayName": "Compression Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "acks": { "name": "acks", "displayName": "Delivery Guarantee", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Demarcator": { "name": "Message Demarcator", "displayName": "Message Demarcator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key Attribute Encoding": { "name": "Kafka Key Attribute Encoding", "displayName": "Kafka Key Attribute Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Metadata Strategy": { "name": "Record Metadata Strategy", "displayName": "Record Metadata Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transactional ID Prefix": { "name": "Transactional ID Prefix", "displayName": "Transactional ID Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Header Encoding": { "name": "Header Encoding", "displayName": "Header Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.request.size": { "name": "max.request.size", "displayName": "Max Request Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Topic Name": { "name": "Topic Name", "displayName": "Topic Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key": { "name": "Kafka Key", "displayName": "Kafka Key", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partition": { "name": "partition", "displayName": "Partition", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Connection Service": { "name": "Kafka Connection Service", "displayName": "Kafka Connection Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Publish Strategy": { "name": "Publish Strategy", "displayName": "Publish Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Key Writer": { "name": "Record Key Writer", "displayName": "Record Key Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Failure Strategy": { "name": "Failure Strategy", "displayName": "Failure Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partitioner.class": { "name": "partitioner.class", "displayName": "Partitioner Class", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Transactions Enabled": { "name": "Transactions Enabled", "displayName": "Transactions Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Key Field": { "name": "Message Key Field", "displayName": "Message Key Field", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "FlowFile Attribute Header Pattern": { "name": "FlowFile Attribute Header Pattern", "displayName": "FlowFile Attribute Header Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "success", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4" } ], "inputPorts": [], "outputPorts": [], "connections": [ { "identifier": "4a16d5e1-b4fa-3018-ba09-504a16238421", "instanceIdentifier": "64b996e2-f357-34fe-aee6-e24d6fd201e4", "name": "", "source": { "id": "e29d8c86-db41-3f3e-9ba7-bffeaefcade6", "type": "PROCESSOR", "groupId": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4", "name": "InvokeHTTP", "comments": "", "instanceIdentifier": "d86d6b2e-ed8f-3d61-9200-5c02c0918eec" }, "destination": { "id": "6f74b992-9f25-3fa8-ada3-eebe335159b3", "type": "PROCESSOR", "groupId": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4", "name": "SplitRecord", "comments": "", "instanceIdentifier": "a5301fb3-0195-1000-0000-0000734e0ae7" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 10000, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4" }, { "identifier": "1025098e-3c22-37c0-9560-203bc51ef427", "instanceIdentifier": "a530ccd7-0195-1000-0000-00004ed00e40", "name": "", "source": { "id": "6f74b992-9f25-3fa8-ada3-eebe335159b3", "type": "PROCESSOR", "groupId": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4", "name": "SplitRecord", "comments": "", "instanceIdentifier": "a5301fb3-0195-1000-0000-0000734e0ae7" }, "destination": { "id": "cd097bd8-d55f-3685-b63f-56d46d14c251", "type": "PROCESSOR", "groupId": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4", "name": "PublishKafka", "comments": "", "instanceIdentifier": "2ab64ebf-3492-348b-8b9a-7a97e920220f" }, "labelIndex": 0, "zIndex": 0, "selectedRelationships": [ "splits" ], "backPressureObjectThreshold": 10000, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4" } ], "labels": [], "funnels": [], "controllerServices": [ { "identifier": "db258647-11aa-30c5-a8c6-a49a4296cf19", "instanceIdentifier": "d22c6c6b-4cd7-354f-b5d4-ba93b975ec64", "name": "Kafka3ConnectionService", "comments": "", "type": "org.apache.nifi.kafka.service.Kafka3ConnectionService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-3-service-nar", "version": "2.6.0" }, "properties": { "security.protocol": "SSL", "SSL Context Service": "7fce889c-fc16-3f69-b7b4-44a90a81ae38", "ack.wait.time": "5 sec", "sasl.username": null, "max.poll.records": "10000", "sasl.mechanism": "GSSAPI", "isolation.level": "read_committed", "max.block.ms": "5 sec", "bootstrap.servers": "kafka-broker-default-headless.default.svc.cluster.local:9093", "kerberos-user-service": null, "sasl.kerberos.service.name": null, "default.api.timeout.ms": "60 sec" }, "propertyDescriptors": { "ack.wait.time": { "name": "ack.wait.time", "displayName": "Acknowledgment Wait Time", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "sasl.username": { "name": "sasl.username", "displayName": "SASL Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.poll.records": { "name": "max.poll.records", "displayName": "Max Poll Records", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "bootstrap.servers": { "name": "bootstrap.servers", "displayName": "Bootstrap Servers", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "sasl.kerberos.service.name": { "name": "sasl.kerberos.service.name", "displayName": "Kerberos Service Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "security.protocol": { "name": "security.protocol", "displayName": "Security Protocol", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "sasl.mechanism": { "name": "sasl.mechanism", "displayName": "SASL Mechanism", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "isolation.level": { "name": "isolation.level", "displayName": "Transaction Isolation Level", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.block.ms": { "name": "max.block.ms", "displayName": "Max Metadata Wait Time", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "kerberos-user-service": { "name": "kerberos-user-service", "displayName": "Kerberos User Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "sasl.password": { "name": "sasl.password", "displayName": "SASL Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "default.api.timeout.ms": { "name": "default.api.timeout.ms", "displayName": "Client Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "controllerServiceApis": [ { "type": "org.apache.nifi.kafka.service.api.KafkaConnectionService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-service-api-nar", "version": "2.6.0" } } ], "scheduledState": "DISABLED", "bulletinLevel": "WARN", "componentType": "CONTROLLER_SERVICE", "groupIdentifier": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4" }, { "identifier": "d082021f-5f72-322a-8e1d-c5990d04b2b1", "instanceIdentifier": "ab119a98-0d96-3556-b54f-411fd9cdd462", "name": "CSVReader", "type": "org.apache.nifi.csv.CSVReader", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-record-serialization-services-nar", "version": "2.6.0" }, "properties": { "schema-reference-reader": null, "ignore-csv-header": "false", "schema-branch": null, "Trim double quote": "true", "CSV Format": "custom", "Quote Character": "\"", "csvutils-allow-duplicate-header-names": "true", "Value Separator": ",", "Record Separator": "\\n", "Timestamp Format": null, "Escape Character": "\\", "Date Format": null, "Null String": null, "Skip Header Line": "false", "Trim Fields": "true", "schema-name": "${schema.name}", "schema-registry": null, "csv-reader-csv-parser": "commons-csv", "Time Format": null, "Comment Marker": null, "schema-access-strategy": "infer-schema", "schema-version": null, "csvutils-character-set": "UTF-8", "schema-text": "${avro.schema}" }, "propertyDescriptors": { "schema-reference-reader": { "name": "schema-reference-reader", "displayName": "Schema Reference Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "ignore-csv-header": { "name": "ignore-csv-header", "displayName": "Ignore CSV Header Column Names", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-branch": { "name": "schema-branch", "displayName": "Schema Branch", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Trim double quote": { "name": "Trim double quote", "displayName": "Trim double quote", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "CSV Format": { "name": "CSV Format", "displayName": "CSV Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Quote Character": { "name": "Quote Character", "displayName": "Quote Character", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "csvutils-allow-duplicate-header-names": { "name": "csvutils-allow-duplicate-header-names", "displayName": "Allow Duplicate Header Names", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Value Separator": { "name": "Value Separator", "displayName": "Value Separator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Separator": { "name": "Record Separator", "displayName": "Record Separator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Timestamp Format": { "name": "Timestamp Format", "displayName": "Timestamp Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Escape Character": { "name": "Escape Character", "displayName": "Escape Character", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Date Format": { "name": "Date Format", "displayName": "Date Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Null String": { "name": "Null String", "displayName": "Null String", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Skip Header Line": { "name": "Skip Header Line", "displayName": "Treat First Line as Header", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Trim Fields": { "name": "Trim Fields", "displayName": "Trim Fields", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-name": { "name": "schema-name", "displayName": "Schema Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-registry": { "name": "schema-registry", "displayName": "Schema Registry", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "csv-reader-csv-parser": { "name": "csv-reader-csv-parser", "displayName": "CSV Parser", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Time Format": { "name": "Time Format", "displayName": "Time Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Comment Marker": { "name": "Comment Marker", "displayName": "Comment Marker", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-access-strategy": { "name": "schema-access-strategy", "displayName": "Schema Access Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-version": { "name": "schema-version", "displayName": "Schema Version", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "csvutils-character-set": { "name": "csvutils-character-set", "displayName": "Character Set", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-text": { "name": "schema-text", "displayName": "Schema Text", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "controllerServiceApis": [ { "type": "org.apache.nifi.serialization.RecordReaderFactory", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } } ], "scheduledState": "DISABLED", "bulletinLevel": "WARN", "componentType": "CONTROLLER_SERVICE", "groupIdentifier": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4" }, { "identifier": "7fce889c-fc16-3f69-b7b4-44a90a81ae38", "instanceIdentifier": "3238df43-ee04-3bec-aa9b-13efe7c0f72a", "name": "StandardRestrictedSSLContextService", "comments": "", "type": "org.apache.nifi.ssl.StandardRestrictedSSLContextService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-ssl-context-service-nar", "version": "2.6.0" }, "properties": { "Truststore Type": "PKCS12", "Truststore Password": "secret", "SSL Protocol": "TLSv1.3", "Keystore Type": "PKCS12", "Keystore Password": "secret", "Truststore Filename": "/stackable/keystore/truststore.p12", "Keystore Filename": "/stackable/keystore/keystore.p12" }, "propertyDescriptors": { "Truststore Type": { "name": "Truststore Type", "displayName": "Truststore Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Protocol": { "name": "SSL Protocol", "displayName": "TLS Protocol", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Keystore Type": { "name": "Keystore Type", "displayName": "Keystore Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Truststore Filename": { "name": "Truststore Filename", "displayName": "Truststore Filename", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "SINGLE", "resourceTypes": [ "FILE" ] } }, "Keystore Password": { "name": "Keystore Password", "displayName": "Keystore Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "key-password": { "name": "key-password", "displayName": "Key Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Truststore Password": { "name": "Truststore Password", "displayName": "Truststore Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Keystore Filename": { "name": "Keystore Filename", "displayName": "Keystore Filename", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "SINGLE", "resourceTypes": [ "FILE" ] } } }, "controllerServiceApis": [ { "type": "org.apache.nifi.ssl.SSLContextService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } }, { "type": "org.apache.nifi.ssl.RestrictedSSLContextService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } }, { "type": "org.apache.nifi.ssl.SSLContextProvider", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } } ], "scheduledState": "DISABLED", "bulletinLevel": "WARN", "componentType": "CONTROLLER_SERVICE", "groupIdentifier": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4" }, { "identifier": "f3254a5e-0024-31c3-b59a-9edff2a099c9", "instanceIdentifier": "8dd599ec-3c90-3a34-8213-185a6d1bfb92", "name": "JsonRecordSetWriter", "type": "org.apache.nifi.json.JsonRecordSetWriter", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-record-serialization-services-nar", "version": "2.6.0" }, "properties": { "schema-reference-reader": null, "schema-branch": null, "Allow Scientific Notation": "false", "compression-level": "1", "schema-cache": null, "Timestamp Format": null, "Date Format": null, "Pretty Print JSON": "false", "compression-format": "none", "Schema Write Strategy": "no-schema", "suppress-nulls": "never-suppress", "output-grouping": "output-array", "schema-name": "${schema.name}", "schema-registry": null, "Time Format": null, "schema-access-strategy": "inherit-record-schema", "schema-version": null, "schema-text": "${avro.schema}", "Schema Reference Writer": null }, "propertyDescriptors": { "schema-reference-reader": { "name": "schema-reference-reader", "displayName": "Schema Reference Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "schema-branch": { "name": "schema-branch", "displayName": "Schema Branch", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Allow Scientific Notation": { "name": "Allow Scientific Notation", "displayName": "Allow Scientific Notation", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "compression-level": { "name": "compression-level", "displayName": "Compression Level", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-cache": { "name": "schema-cache", "displayName": "Schema Cache", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Timestamp Format": { "name": "Timestamp Format", "displayName": "Timestamp Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Date Format": { "name": "Date Format", "displayName": "Date Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Pretty Print JSON": { "name": "Pretty Print JSON", "displayName": "Pretty Print JSON", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "compression-format": { "name": "compression-format", "displayName": "Compression Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Schema Write Strategy": { "name": "Schema Write Strategy", "displayName": "Schema Write Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "suppress-nulls": { "name": "suppress-nulls", "displayName": "Suppress Null Values", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "output-grouping": { "name": "output-grouping", "displayName": "Output Grouping", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-name": { "name": "schema-name", "displayName": "Schema Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-registry": { "name": "schema-registry", "displayName": "Schema Registry", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Time Format": { "name": "Time Format", "displayName": "Time Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-access-strategy": { "name": "schema-access-strategy", "displayName": "Schema Access Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-version": { "name": "schema-version", "displayName": "Schema Version", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-text": { "name": "schema-text", "displayName": "Schema Text", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Schema Reference Writer": { "name": "Schema Reference Writer", "displayName": "Schema Reference Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false } }, "controllerServiceApis": [ { "type": "org.apache.nifi.serialization.RecordSetWriterFactory", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } } ], "scheduledState": "DISABLED", "bulletinLevel": "WARN", "componentType": "CONTROLLER_SERVICE", "groupIdentifier": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4" }, { "identifier": "5e81a80d-728d-3c87-800e-aa957618da55", "instanceIdentifier": "a53045c6-0195-1000-ffff-ffffef6c13db", "name": "CSVRecordSetWriter", "comments": "", "type": "org.apache.nifi.csv.CSVRecordSetWriter", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-record-serialization-services-nar", "version": "2.6.0" }, "properties": { "schema-reference-reader": null, "schema-branch": null, "CSV Format": "custom", "Include Header Line": "true", "schema-cache": null, "Escape Character": "\\", "Date Format": null, "Null String": null, "Trim Fields": "true", "schema-registry": null, "Time Format": null, "Comment Marker": null, "schema-access-strategy": "inherit-record-schema", "schema-version": null, "csvutils-character-set": "UTF-8", "schema-text": "${avro.schema}", "csv-writer": "commons-csv", "Quote Character": "\"", "Value Separator": ",", "Record Separator": "\\n", "Timestamp Format": null, "Schema Write Strategy": "no-schema", "schema-name": "${schema.name}", "Quote Mode": "MINIMAL", "Schema Reference Writer": null, "Include Trailing Delimiter": "false" }, "propertyDescriptors": { "schema-reference-reader": { "name": "schema-reference-reader", "displayName": "Schema Reference Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "schema-branch": { "name": "schema-branch", "displayName": "Schema Branch", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "CSV Format": { "name": "CSV Format", "displayName": "CSV Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Include Header Line": { "name": "Include Header Line", "displayName": "Include Header Line", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-cache": { "name": "schema-cache", "displayName": "Schema Cache", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Escape Character": { "name": "Escape Character", "displayName": "Escape Character", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Date Format": { "name": "Date Format", "displayName": "Date Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Null String": { "name": "Null String", "displayName": "Null String", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Trim Fields": { "name": "Trim Fields", "displayName": "Trim Fields", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-registry": { "name": "schema-registry", "displayName": "Schema Registry", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Time Format": { "name": "Time Format", "displayName": "Time Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Comment Marker": { "name": "Comment Marker", "displayName": "Comment Marker", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-access-strategy": { "name": "schema-access-strategy", "displayName": "Schema Access Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-version": { "name": "schema-version", "displayName": "Schema Version", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "csvutils-character-set": { "name": "csvutils-character-set", "displayName": "Character Set", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-text": { "name": "schema-text", "displayName": "Schema Text", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "csv-writer": { "name": "csv-writer", "displayName": "CSV Writer", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Quote Character": { "name": "Quote Character", "displayName": "Quote Character", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Value Separator": { "name": "Value Separator", "displayName": "Value Separator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Separator": { "name": "Record Separator", "displayName": "Record Separator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Timestamp Format": { "name": "Timestamp Format", "displayName": "Timestamp Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Schema Write Strategy": { "name": "Schema Write Strategy", "displayName": "Schema Write Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-name": { "name": "schema-name", "displayName": "Schema Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Quote Mode": { "name": "Quote Mode", "displayName": "Quote Mode", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Schema Reference Writer": { "name": "Schema Reference Writer", "displayName": "Schema Reference Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Include Trailing Delimiter": { "name": "Include Trailing Delimiter", "displayName": "Include Trailing Delimiter", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "controllerServiceApis": [ { "type": "org.apache.nifi.serialization.RecordSetWriterFactory", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } } ], "scheduledState": "DISABLED", "bulletinLevel": "WARN", "componentType": "CONTROLLER_SERVICE", "groupIdentifier": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4" } ], "defaultFlowFileExpiration": "0 sec", "defaultBackPressureObjectThreshold": 10000, "defaultBackPressureDataSizeThreshold": "1 GB", "scheduledState": "ENABLED", "executionEngine": "INHERITED", "maxConcurrentTasks": 1, "statelessFlowTimeout": "1 min", "flowFileConcurrency": "UNBOUNDED", "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE", "componentType": "PROCESS_GROUP" }, "externalControllerServices": {}, "parameterContexts": {}, "flowEncodingVersion": "1.0", "parameterProviders": {}, "latest": false } + { "flowContents": { "identifier": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4", "instanceIdentifier": "a52fc536-0195-1000-ffff-ffffb77a46e7", "name": "IngestEarthquakesToKafka_raw", "comments": "", "position": { "x": -408.0, "y": -176.0 }, "processGroups": [], "remoteProcessGroups": [], "processors": [ { "identifier": "6f74b992-9f25-3fa8-ada3-eebe335159b3", "instanceIdentifier": "a5301fb3-0195-1000-0000-0000734e0ae7", "name": "SplitRecord", "comments": "", "position": { "x": 472.0, "y": 376.0 }, "type": "org.apache.nifi.processors.standard.SplitRecord", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Record Writer": "5e81a80d-728d-3c87-800e-aa957618da55", "Record Reader": "d082021f-5f72-322a-8e1d-c5990d04b2b1", "Records Per Split": "2000" }, "propertyDescriptors": { "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Records Per Split": { "name": "Records Per Split", "displayName": "Records Per Split", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "original", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4" }, { "identifier": "e29d8c86-db41-3f3e-9ba7-bffeaefcade6", "instanceIdentifier": "d86d6b2e-ed8f-3d61-9200-5c02c0918eec", "name": "InvokeHTTP", "comments": "", "position": { "x": 473.00000076287904, "y": 142.99998844850802 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://repo.stackable.tech/repository/misc/earthquake-data/earthquakes_1950_to_2022.csv", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "true", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "365000 days", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "PRIMARY", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4" }, { "identifier": "cd097bd8-d55f-3685-b63f-56d46d14c251", "instanceIdentifier": "2ab64ebf-3492-348b-8b9a-7a97e920220f", "name": "PublishKafka", "comments": "", "position": { "x": 472.0, "y": 608.0 }, "type": "org.apache.nifi.kafka.processors.PublishKafka", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-nar", "version": "2.6.0" }, "properties": { "compression.type": "none", "acks": "all", "Message Demarcator": null, "Kafka Key Attribute Encoding": "utf-8", "Record Reader": "d082021f-5f72-322a-8e1d-c5990d04b2b1", "Record Metadata Strategy": "FROM_PROPERTIES", "Transactional ID Prefix": null, "Header Encoding": "UTF-8", "max.request.size": "1 MB", "Topic Name": "earthquakes", "Kafka Key": null, "partition": null, "Kafka Connection Service": "db258647-11aa-30c5-a8c6-a49a4296cf19", "Publish Strategy": "USE_VALUE", "Record Key Writer": null, "Failure Strategy": "Route to Failure", "partitioner.class": "org.apache.kafka.clients.producer.internals.DefaultPartitioner", "Record Writer": "f3254a5e-0024-31c3-b59a-9edff2a099c9", "Transactions Enabled": "false", "Message Key Field": null, "FlowFile Attribute Header Pattern": null }, "propertyDescriptors": { "compression.type": { "name": "compression.type", "displayName": "Compression Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "acks": { "name": "acks", "displayName": "Delivery Guarantee", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Demarcator": { "name": "Message Demarcator", "displayName": "Message Demarcator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key Attribute Encoding": { "name": "Kafka Key Attribute Encoding", "displayName": "Kafka Key Attribute Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Metadata Strategy": { "name": "Record Metadata Strategy", "displayName": "Record Metadata Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transactional ID Prefix": { "name": "Transactional ID Prefix", "displayName": "Transactional ID Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Header Encoding": { "name": "Header Encoding", "displayName": "Header Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.request.size": { "name": "max.request.size", "displayName": "Max Request Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Topic Name": { "name": "Topic Name", "displayName": "Topic Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key": { "name": "Kafka Key", "displayName": "Kafka Key", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partition": { "name": "partition", "displayName": "Partition", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Connection Service": { "name": "Kafka Connection Service", "displayName": "Kafka Connection Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Publish Strategy": { "name": "Publish Strategy", "displayName": "Publish Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Key Writer": { "name": "Record Key Writer", "displayName": "Record Key Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Failure Strategy": { "name": "Failure Strategy", "displayName": "Failure Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partitioner.class": { "name": "partitioner.class", "displayName": "Partitioner Class", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Transactions Enabled": { "name": "Transactions Enabled", "displayName": "Transactions Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Key Field": { "name": "Message Key Field", "displayName": "Message Key Field", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "FlowFile Attribute Header Pattern": { "name": "FlowFile Attribute Header Pattern", "displayName": "FlowFile Attribute Header Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "success", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4" } ], "inputPorts": [], "outputPorts": [], "connections": [ { "identifier": "4a16d5e1-b4fa-3018-ba09-504a16238421", "instanceIdentifier": "64b996e2-f357-34fe-aee6-e24d6fd201e4", "name": "", "source": { "id": "e29d8c86-db41-3f3e-9ba7-bffeaefcade6", "type": "PROCESSOR", "groupId": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4", "name": "InvokeHTTP", "comments": "", "instanceIdentifier": "d86d6b2e-ed8f-3d61-9200-5c02c0918eec" }, "destination": { "id": "6f74b992-9f25-3fa8-ada3-eebe335159b3", "type": "PROCESSOR", "groupId": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4", "name": "SplitRecord", "comments": "", "instanceIdentifier": "a5301fb3-0195-1000-0000-0000734e0ae7" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 10000, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4" }, { "identifier": "1025098e-3c22-37c0-9560-203bc51ef427", "instanceIdentifier": "a530ccd7-0195-1000-0000-00004ed00e40", "name": "", "source": { "id": "6f74b992-9f25-3fa8-ada3-eebe335159b3", "type": "PROCESSOR", "groupId": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4", "name": "SplitRecord", "comments": "", "instanceIdentifier": "a5301fb3-0195-1000-0000-0000734e0ae7" }, "destination": { "id": "cd097bd8-d55f-3685-b63f-56d46d14c251", "type": "PROCESSOR", "groupId": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4", "name": "PublishKafka", "comments": "", "instanceIdentifier": "2ab64ebf-3492-348b-8b9a-7a97e920220f" }, "labelIndex": 0, "zIndex": 0, "selectedRelationships": [ "splits" ], "backPressureObjectThreshold": 10000, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4" } ], "labels": [], "funnels": [], "controllerServices": [ { "identifier": "db258647-11aa-30c5-a8c6-a49a4296cf19", "instanceIdentifier": "d22c6c6b-4cd7-354f-b5d4-ba93b975ec64", "name": "Kafka3ConnectionService", "comments": "", "type": "org.apache.nifi.kafka.service.Kafka3ConnectionService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-3-service-nar", "version": "2.6.0" }, "properties": { "security.protocol": "SSL", "SSL Context Service": "7fce889c-fc16-3f69-b7b4-44a90a81ae38", "ack.wait.time": "5 sec", "sasl.username": null, "max.poll.records": "10000", "sasl.mechanism": "GSSAPI", "isolation.level": "read_committed", "max.block.ms": "5 sec", "bootstrap.servers": "kafka-broker-default-headless.{{ NAMESPACE }}.svc.cluster.local:9093", "kerberos-user-service": null, "sasl.kerberos.service.name": null, "default.api.timeout.ms": "60 sec" }, "propertyDescriptors": { "ack.wait.time": { "name": "ack.wait.time", "displayName": "Acknowledgment Wait Time", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "sasl.username": { "name": "sasl.username", "displayName": "SASL Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.poll.records": { "name": "max.poll.records", "displayName": "Max Poll Records", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "bootstrap.servers": { "name": "bootstrap.servers", "displayName": "Bootstrap Servers", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "sasl.kerberos.service.name": { "name": "sasl.kerberos.service.name", "displayName": "Kerberos Service Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "security.protocol": { "name": "security.protocol", "displayName": "Security Protocol", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "sasl.mechanism": { "name": "sasl.mechanism", "displayName": "SASL Mechanism", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "isolation.level": { "name": "isolation.level", "displayName": "Transaction Isolation Level", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.block.ms": { "name": "max.block.ms", "displayName": "Max Metadata Wait Time", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "kerberos-user-service": { "name": "kerberos-user-service", "displayName": "Kerberos User Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "sasl.password": { "name": "sasl.password", "displayName": "SASL Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "default.api.timeout.ms": { "name": "default.api.timeout.ms", "displayName": "Client Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "controllerServiceApis": [ { "type": "org.apache.nifi.kafka.service.api.KafkaConnectionService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-service-api-nar", "version": "2.6.0" } } ], "scheduledState": "DISABLED", "bulletinLevel": "WARN", "componentType": "CONTROLLER_SERVICE", "groupIdentifier": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4" }, { "identifier": "d082021f-5f72-322a-8e1d-c5990d04b2b1", "instanceIdentifier": "ab119a98-0d96-3556-b54f-411fd9cdd462", "name": "CSVReader", "type": "org.apache.nifi.csv.CSVReader", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-record-serialization-services-nar", "version": "2.6.0" }, "properties": { "schema-reference-reader": null, "ignore-csv-header": "false", "schema-branch": null, "Trim double quote": "true", "CSV Format": "custom", "Quote Character": "\"", "csvutils-allow-duplicate-header-names": "true", "Value Separator": ",", "Record Separator": "\\n", "Timestamp Format": null, "Escape Character": "\\", "Date Format": null, "Null String": null, "Skip Header Line": "false", "Trim Fields": "true", "schema-name": "${schema.name}", "schema-registry": null, "csv-reader-csv-parser": "commons-csv", "Time Format": null, "Comment Marker": null, "schema-access-strategy": "infer-schema", "schema-version": null, "csvutils-character-set": "UTF-8", "schema-text": "${avro.schema}" }, "propertyDescriptors": { "schema-reference-reader": { "name": "schema-reference-reader", "displayName": "Schema Reference Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "ignore-csv-header": { "name": "ignore-csv-header", "displayName": "Ignore CSV Header Column Names", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-branch": { "name": "schema-branch", "displayName": "Schema Branch", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Trim double quote": { "name": "Trim double quote", "displayName": "Trim double quote", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "CSV Format": { "name": "CSV Format", "displayName": "CSV Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Quote Character": { "name": "Quote Character", "displayName": "Quote Character", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "csvutils-allow-duplicate-header-names": { "name": "csvutils-allow-duplicate-header-names", "displayName": "Allow Duplicate Header Names", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Value Separator": { "name": "Value Separator", "displayName": "Value Separator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Separator": { "name": "Record Separator", "displayName": "Record Separator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Timestamp Format": { "name": "Timestamp Format", "displayName": "Timestamp Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Escape Character": { "name": "Escape Character", "displayName": "Escape Character", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Date Format": { "name": "Date Format", "displayName": "Date Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Null String": { "name": "Null String", "displayName": "Null String", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Skip Header Line": { "name": "Skip Header Line", "displayName": "Treat First Line as Header", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Trim Fields": { "name": "Trim Fields", "displayName": "Trim Fields", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-name": { "name": "schema-name", "displayName": "Schema Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-registry": { "name": "schema-registry", "displayName": "Schema Registry", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "csv-reader-csv-parser": { "name": "csv-reader-csv-parser", "displayName": "CSV Parser", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Time Format": { "name": "Time Format", "displayName": "Time Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Comment Marker": { "name": "Comment Marker", "displayName": "Comment Marker", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-access-strategy": { "name": "schema-access-strategy", "displayName": "Schema Access Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-version": { "name": "schema-version", "displayName": "Schema Version", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "csvutils-character-set": { "name": "csvutils-character-set", "displayName": "Character Set", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-text": { "name": "schema-text", "displayName": "Schema Text", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "controllerServiceApis": [ { "type": "org.apache.nifi.serialization.RecordReaderFactory", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } } ], "scheduledState": "DISABLED", "bulletinLevel": "WARN", "componentType": "CONTROLLER_SERVICE", "groupIdentifier": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4" }, { "identifier": "7fce889c-fc16-3f69-b7b4-44a90a81ae38", "instanceIdentifier": "3238df43-ee04-3bec-aa9b-13efe7c0f72a", "name": "StandardRestrictedSSLContextService", "comments": "", "type": "org.apache.nifi.ssl.StandardRestrictedSSLContextService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-ssl-context-service-nar", "version": "2.6.0" }, "properties": { "Truststore Type": "PKCS12", "Truststore Password": "secret", "SSL Protocol": "TLSv1.3", "Keystore Type": "PKCS12", "Keystore Password": "secret", "Truststore Filename": "/stackable/keystore/truststore.p12", "Keystore Filename": "/stackable/keystore/keystore.p12" }, "propertyDescriptors": { "Truststore Type": { "name": "Truststore Type", "displayName": "Truststore Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Protocol": { "name": "SSL Protocol", "displayName": "TLS Protocol", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Keystore Type": { "name": "Keystore Type", "displayName": "Keystore Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Truststore Filename": { "name": "Truststore Filename", "displayName": "Truststore Filename", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "SINGLE", "resourceTypes": [ "FILE" ] } }, "Keystore Password": { "name": "Keystore Password", "displayName": "Keystore Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "key-password": { "name": "key-password", "displayName": "Key Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Truststore Password": { "name": "Truststore Password", "displayName": "Truststore Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Keystore Filename": { "name": "Keystore Filename", "displayName": "Keystore Filename", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "SINGLE", "resourceTypes": [ "FILE" ] } } }, "controllerServiceApis": [ { "type": "org.apache.nifi.ssl.SSLContextService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } }, { "type": "org.apache.nifi.ssl.RestrictedSSLContextService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } }, { "type": "org.apache.nifi.ssl.SSLContextProvider", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } } ], "scheduledState": "DISABLED", "bulletinLevel": "WARN", "componentType": "CONTROLLER_SERVICE", "groupIdentifier": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4" }, { "identifier": "f3254a5e-0024-31c3-b59a-9edff2a099c9", "instanceIdentifier": "8dd599ec-3c90-3a34-8213-185a6d1bfb92", "name": "JsonRecordSetWriter", "type": "org.apache.nifi.json.JsonRecordSetWriter", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-record-serialization-services-nar", "version": "2.6.0" }, "properties": { "schema-reference-reader": null, "schema-branch": null, "Allow Scientific Notation": "false", "compression-level": "1", "schema-cache": null, "Timestamp Format": null, "Date Format": null, "Pretty Print JSON": "false", "compression-format": "none", "Schema Write Strategy": "no-schema", "suppress-nulls": "never-suppress", "output-grouping": "output-array", "schema-name": "${schema.name}", "schema-registry": null, "Time Format": null, "schema-access-strategy": "inherit-record-schema", "schema-version": null, "schema-text": "${avro.schema}", "Schema Reference Writer": null }, "propertyDescriptors": { "schema-reference-reader": { "name": "schema-reference-reader", "displayName": "Schema Reference Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "schema-branch": { "name": "schema-branch", "displayName": "Schema Branch", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Allow Scientific Notation": { "name": "Allow Scientific Notation", "displayName": "Allow Scientific Notation", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "compression-level": { "name": "compression-level", "displayName": "Compression Level", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-cache": { "name": "schema-cache", "displayName": "Schema Cache", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Timestamp Format": { "name": "Timestamp Format", "displayName": "Timestamp Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Date Format": { "name": "Date Format", "displayName": "Date Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Pretty Print JSON": { "name": "Pretty Print JSON", "displayName": "Pretty Print JSON", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "compression-format": { "name": "compression-format", "displayName": "Compression Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Schema Write Strategy": { "name": "Schema Write Strategy", "displayName": "Schema Write Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "suppress-nulls": { "name": "suppress-nulls", "displayName": "Suppress Null Values", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "output-grouping": { "name": "output-grouping", "displayName": "Output Grouping", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-name": { "name": "schema-name", "displayName": "Schema Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-registry": { "name": "schema-registry", "displayName": "Schema Registry", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Time Format": { "name": "Time Format", "displayName": "Time Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-access-strategy": { "name": "schema-access-strategy", "displayName": "Schema Access Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-version": { "name": "schema-version", "displayName": "Schema Version", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-text": { "name": "schema-text", "displayName": "Schema Text", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Schema Reference Writer": { "name": "Schema Reference Writer", "displayName": "Schema Reference Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false } }, "controllerServiceApis": [ { "type": "org.apache.nifi.serialization.RecordSetWriterFactory", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } } ], "scheduledState": "DISABLED", "bulletinLevel": "WARN", "componentType": "CONTROLLER_SERVICE", "groupIdentifier": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4" }, { "identifier": "5e81a80d-728d-3c87-800e-aa957618da55", "instanceIdentifier": "a53045c6-0195-1000-ffff-ffffef6c13db", "name": "CSVRecordSetWriter", "comments": "", "type": "org.apache.nifi.csv.CSVRecordSetWriter", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-record-serialization-services-nar", "version": "2.6.0" }, "properties": { "schema-reference-reader": null, "schema-branch": null, "CSV Format": "custom", "Include Header Line": "true", "schema-cache": null, "Escape Character": "\\", "Date Format": null, "Null String": null, "Trim Fields": "true", "schema-registry": null, "Time Format": null, "Comment Marker": null, "schema-access-strategy": "inherit-record-schema", "schema-version": null, "csvutils-character-set": "UTF-8", "schema-text": "${avro.schema}", "csv-writer": "commons-csv", "Quote Character": "\"", "Value Separator": ",", "Record Separator": "\\n", "Timestamp Format": null, "Schema Write Strategy": "no-schema", "schema-name": "${schema.name}", "Quote Mode": "MINIMAL", "Schema Reference Writer": null, "Include Trailing Delimiter": "false" }, "propertyDescriptors": { "schema-reference-reader": { "name": "schema-reference-reader", "displayName": "Schema Reference Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "schema-branch": { "name": "schema-branch", "displayName": "Schema Branch", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "CSV Format": { "name": "CSV Format", "displayName": "CSV Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Include Header Line": { "name": "Include Header Line", "displayName": "Include Header Line", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-cache": { "name": "schema-cache", "displayName": "Schema Cache", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Escape Character": { "name": "Escape Character", "displayName": "Escape Character", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Date Format": { "name": "Date Format", "displayName": "Date Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Null String": { "name": "Null String", "displayName": "Null String", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Trim Fields": { "name": "Trim Fields", "displayName": "Trim Fields", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-registry": { "name": "schema-registry", "displayName": "Schema Registry", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Time Format": { "name": "Time Format", "displayName": "Time Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Comment Marker": { "name": "Comment Marker", "displayName": "Comment Marker", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-access-strategy": { "name": "schema-access-strategy", "displayName": "Schema Access Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-version": { "name": "schema-version", "displayName": "Schema Version", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "csvutils-character-set": { "name": "csvutils-character-set", "displayName": "Character Set", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-text": { "name": "schema-text", "displayName": "Schema Text", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "csv-writer": { "name": "csv-writer", "displayName": "CSV Writer", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Quote Character": { "name": "Quote Character", "displayName": "Quote Character", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Value Separator": { "name": "Value Separator", "displayName": "Value Separator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Separator": { "name": "Record Separator", "displayName": "Record Separator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Timestamp Format": { "name": "Timestamp Format", "displayName": "Timestamp Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Schema Write Strategy": { "name": "Schema Write Strategy", "displayName": "Schema Write Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-name": { "name": "schema-name", "displayName": "Schema Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Quote Mode": { "name": "Quote Mode", "displayName": "Quote Mode", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Schema Reference Writer": { "name": "Schema Reference Writer", "displayName": "Schema Reference Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Include Trailing Delimiter": { "name": "Include Trailing Delimiter", "displayName": "Include Trailing Delimiter", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "controllerServiceApis": [ { "type": "org.apache.nifi.serialization.RecordSetWriterFactory", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } } ], "scheduledState": "DISABLED", "bulletinLevel": "WARN", "componentType": "CONTROLLER_SERVICE", "groupIdentifier": "a1618dd5-e11a-3ff4-9a6c-62607956b4f4" } ], "defaultFlowFileExpiration": "0 sec", "defaultBackPressureObjectThreshold": 10000, "defaultBackPressureDataSizeThreshold": "1 GB", "scheduledState": "ENABLED", "executionEngine": "INHERITED", "maxConcurrentTasks": 1, "statelessFlowTimeout": "1 min", "flowFileConcurrency": "UNBOUNDED", "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE", "componentType": "PROCESS_GROUP" }, "externalControllerServices": {}, "parameterContexts": {}, "flowEncodingVersion": "1.0", "parameterProviders": {}, "latest": false } diff --git a/demos/nifi-kafka-druid-earthquake-data/serviceaccount.yaml b/demos/nifi-kafka-druid-earthquake-data/serviceaccount.yaml index 8a2d0b19..d7c0b939 100644 --- a/demos/nifi-kafka-druid-earthquake-data/serviceaccount.yaml +++ b/demos/nifi-kafka-druid-earthquake-data/serviceaccount.yaml @@ -3,7 +3,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: demo-serviceaccount - namespace: default --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -12,7 +11,7 @@ metadata: subjects: - kind: ServiceAccount name: demo-serviceaccount - namespace: default + namespace: {{ NAMESPACE }} roleRef: kind: ClusterRole name: demo-clusterrole diff --git a/stacks/_templates/postgresql-druid.yaml b/stacks/_templates/postgresql-druid.yaml index b181c92e..96ae74a3 100644 --- a/stacks/_templates/postgresql-druid.yaml +++ b/stacks/_templates/postgresql-druid.yaml @@ -8,6 +8,7 @@ version: 18.1.3 # 18.0.0 options: commonLabels: stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} auth: username: druid password: druid From f1bb0551274300d4bfe1807f419e287b967f7448 Mon Sep 17 00:00:00 2001 From: xeniape Date: Mon, 9 Mar 2026 10:45:02 +0100 Subject: [PATCH 04/22] nifi-kafka-druid-water-level-data demo in non-default ns and with demo labels --- demos/demos-v2.yaml | 2 +- .../create-druid-ingestion-job.yaml | 4 ++-- .../create-nifi-ingestion-job.yaml | 3 +-- demos/nifi-kafka-druid-water-level-data/serviceaccount.yaml | 3 +-- .../demos/pages/nifi-kafka-druid-water-level-data.adoc | 2 +- stacks/stacks-v2.yaml | 3 +++ 6 files changed, 9 insertions(+), 8 deletions(-) diff --git a/demos/demos-v2.yaml b/demos/demos-v2.yaml index 5e481664..9204b852 100644 --- a/demos/demos-v2.yaml +++ b/demos/demos-v2.yaml @@ -133,7 +133,7 @@ demos: - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/setup-superset.yaml - supportedNamespaces: ["default"] + supportedNamespaces: [] resourceRequests: cpu: 8900m memory: 42330Mi diff --git a/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml b/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml index e852beb9..ac614a0e 100644 --- a/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml +++ b/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml @@ -54,7 +54,7 @@ data: "ioConfig": { "type": "kafka", "consumerProperties": { - "bootstrap.servers": "kafka-broker-default-headless.default.svc.cluster.local:9093", + "bootstrap.servers": "kafka-broker-default-headless.{{ NAMESPACE }}.svc.cluster.local:9093", "security.protocol": "SSL", "ssl.truststore.location": "/stackable/tls/truststore.p12", "ssl.truststore.password": "changeit", @@ -133,7 +133,7 @@ data: "ioConfig": { "type": "kafka", "consumerProperties": { - "bootstrap.servers": "kafka-broker-default-headless.default.svc.cluster.local:9093", + "bootstrap.servers": "kafka-broker-default-headless.{{ NAMESPACE }}.svc.cluster.local:9093", "security.protocol": "SSL", "ssl.truststore.location": "/stackable/tls/truststore.p12", "ssl.truststore.password": "changeit", diff --git a/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml b/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml index 46796124..28a530f4 100644 --- a/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml +++ b/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml @@ -127,7 +127,6 @@ apiVersion: v1 kind: ConfigMap metadata: name: ingest-water-level-data-flow - namespace: default data: IngestWaterLevelsToKafka.json: | - { "flowContents": { "identifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "instanceIdentifier": "81598f35-0195-1000-0000-000073723729", "name": "IngestWaterLevelsToKafka_raw", "comments": "", "position": { "x": -472.0, "y": -496.0 }, "processGroups": [], "remoteProcessGroups": [], "processors": [ { "identifier": "e85c2e60-192e-3ed8-ab64-750311e0ed45", "instanceIdentifier": "46c0209c-7abe-3a8e-bbc0-39f9333f14b1", "name": "Get station list", "comments": "", "position": { "x": 485.99999797539556, "y": 246.99999837990495 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations.json", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "false", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "365000 days", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "PRIMARY", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "50bc20ed-2b1b-3068-0000-0000423c0a67", "instanceIdentifier": "3c9c0c1f-db51-39f1-bcea-5aa511e75799", "name": "Add station_uuid", "comments": "", "position": { "x": 1240.0, "y": 1192.0 }, "type": "org.apache.nifi.processors.jolt.JoltTransformJSON", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-jolt-nar", "version": "2.6.0" }, "properties": { "Jolt Transform": "jolt-transform-chain", "Max String Length": "20 MB", "Transform Cache Size": "1", "Custom Transformation Class Name": null, "Custom Module Directory": null, "Pretty Print": "false", "Jolt Specification": "[\n {\n \"operation\": \"default\",\n \"spec\": {\n \"*\": {\n \"station_uuid\": \"${station_uuid}\"\n }\n }\n }\n]\n" }, "propertyDescriptors": { "Jolt Transform": { "name": "Jolt Transform", "displayName": "Jolt Transform", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Max String Length": { "name": "Max String Length", "displayName": "Max String Length", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transform Cache Size": { "name": "Transform Cache Size", "displayName": "Transform Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Transformation Class Name": { "name": "Custom Transformation Class Name", "displayName": "Custom Transformation Class Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Module Directory": { "name": "Custom Module Directory", "displayName": "Custom Module Directory", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "MULTIPLE", "resourceTypes": [ "DIRECTORY", "FILE" ] } }, "Pretty Print": { "name": "Pretty Print", "displayName": "Pretty Print", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Jolt Specification": { "name": "Jolt Specification", "displayName": "Jolt Specification", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "SINGLE", "resourceTypes": [ "TEXT", "FILE" ] } } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "8809ece7-fbe0-3613-b7b0-77e05c8f4cb1", "instanceIdentifier": "1425717b-a077-39b0-872e-7381d0b3848f", "name": "Extract station_uuid", "comments": "", "position": { "x": 861.9999979753956, "y": 726.9999983799049 }, "type": "org.apache.nifi.processors.standard.EvaluateJsonPath", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Destination": "flowfile-attribute", "Max String Length": "20 MB", "Return Type": "auto-detect", "Null Value Representation": "empty string", "station_uuid": "$.uuid", "Path Not Found Behavior": "ignore" }, "propertyDescriptors": { "Destination": { "name": "Destination", "displayName": "Destination", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Max String Length": { "name": "Max String Length", "displayName": "Max String Length", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Return Type": { "name": "Return Type", "displayName": "Return Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Null Value Representation": { "name": "Null Value Representation", "displayName": "Null Value Representation", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "station_uuid": { "name": "station_uuid", "displayName": "station_uuid", "identifiesControllerService": false, "sensitive": false, "dynamic": true }, "Path Not Found Behavior": { "name": "Path Not Found Behavior", "displayName": "Path Not Found Behavior", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "failure", "unmatched" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "656d02d2-3100-39e6-a3dd-a8fe91b6dd06", "instanceIdentifier": "cec0a8ff-73d7-35d1-9f9c-0c5cc0af6db2", "name": "SplitRecord", "comments": "", "position": { "x": 1237.9999979753957, "y": 510.9999983799049 }, "type": "org.apache.nifi.processors.standard.SplitRecord", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Record Writer": "f6b7ad0f-da95-39c4-be8d-4ae944d8034b", "Record Reader": "e75c34eb-8e60-3b6b-8a29-3dd7b79776c4", "Records Per Split": "1" }, "propertyDescriptors": { "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Records Per Split": { "name": "Records Per Split", "displayName": "Records Per Split", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "original", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "e2bdb78b-009b-3086-ad39-816874d3be80", "instanceIdentifier": "72294ac8-46e9-33fb-8923-8e6cb4c69269", "name": "Get station list", "comments": "", "position": { "x": 861.9999979753956, "y": 246.99999837990495 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations.json", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "false", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "365000 days", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "PRIMARY", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "dde01269-6b44-3f4c-9406-8e291cd8aaa4", "instanceIdentifier": "816243be-0195-1000-ffff-fffff00620a1", "name": "Add station_uuid", "comments": "", "position": { "x": 864.0, "y": 1192.0 }, "type": "org.apache.nifi.processors.jolt.JoltTransformJSON", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-jolt-nar", "version": "2.6.0" }, "properties": { "Jolt Transform": "jolt-transform-chain", "Max String Length": "20 MB", "Transform Cache Size": "1", "Custom Transformation Class Name": null, "Custom Module Directory": null, "Pretty Print": "false", "Jolt Specification": "[\n {\n \"operation\": \"default\",\n \"spec\": {\n \"*\": {\n \"station_uuid\": \"${station_uuid}\"\n }\n }\n }\n]\n" }, "propertyDescriptors": { "Jolt Transform": { "name": "Jolt Transform", "displayName": "Jolt Transform", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Max String Length": { "name": "Max String Length", "displayName": "Max String Length", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transform Cache Size": { "name": "Transform Cache Size", "displayName": "Transform Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Transformation Class Name": { "name": "Custom Transformation Class Name", "displayName": "Custom Transformation Class Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Module Directory": { "name": "Custom Module Directory", "displayName": "Custom Module Directory", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "MULTIPLE", "resourceTypes": [ "DIRECTORY", "FILE" ] } }, "Pretty Print": { "name": "Pretty Print", "displayName": "Pretty Print", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Jolt Specification": { "name": "Jolt Specification", "displayName": "Jolt Specification", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "SINGLE", "resourceTypes": [ "TEXT", "FILE" ] } } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "98a12d17-c277-3226-af8b-03848dbdbf11", "instanceIdentifier": "816125f7-0195-1000-0000-00006b8c4170", "name": "PublishKafka", "comments": "", "position": { "x": 1240.0, "y": 1408.0 }, "type": "org.apache.nifi.kafka.processors.PublishKafka", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-nar", "version": "2.6.0" }, "properties": { "compression.type": "snappy", "acks": "all", "Message Demarcator": null, "Kafka Key Attribute Encoding": "utf-8", "Record Reader": "e75c34eb-8e60-3b6b-8a29-3dd7b79776c4", "Record Metadata Strategy": "FROM_PROPERTIES", "Transactional ID Prefix": null, "Header Encoding": "UTF-8", "max.request.size": "1 MB", "Topic Name": "measurements", "Kafka Key": null, "partition": null, "Kafka Connection Service": "29032623-5723-3b5d-8983-07534da27fd5", "Publish Strategy": "USE_VALUE", "Record Key Writer": null, "Failure Strategy": "Route to Failure", "partitioner.class": "org.apache.kafka.clients.producer.internals.DefaultPartitioner", "Record Writer": "f6b7ad0f-da95-39c4-be8d-4ae944d8034b", "Transactions Enabled": "false", "Message Key Field": null, "FlowFile Attribute Header Pattern": null }, "propertyDescriptors": { "compression.type": { "name": "compression.type", "displayName": "Compression Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "acks": { "name": "acks", "displayName": "Delivery Guarantee", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Demarcator": { "name": "Message Demarcator", "displayName": "Message Demarcator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key Attribute Encoding": { "name": "Kafka Key Attribute Encoding", "displayName": "Kafka Key Attribute Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Metadata Strategy": { "name": "Record Metadata Strategy", "displayName": "Record Metadata Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transactional ID Prefix": { "name": "Transactional ID Prefix", "displayName": "Transactional ID Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Header Encoding": { "name": "Header Encoding", "displayName": "Header Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.request.size": { "name": "max.request.size", "displayName": "Max Request Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Topic Name": { "name": "Topic Name", "displayName": "Topic Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key": { "name": "Kafka Key", "displayName": "Kafka Key", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partition": { "name": "partition", "displayName": "Partition", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Connection Service": { "name": "Kafka Connection Service", "displayName": "Kafka Connection Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Publish Strategy": { "name": "Publish Strategy", "displayName": "Publish Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Key Writer": { "name": "Record Key Writer", "displayName": "Record Key Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Failure Strategy": { "name": "Failure Strategy", "displayName": "Failure Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partitioner.class": { "name": "partitioner.class", "displayName": "Partitioner Class", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Transactions Enabled": { "name": "Transactions Enabled", "displayName": "Transactions Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Key Field": { "name": "Message Key Field", "displayName": "Message Key Field", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "FlowFile Attribute Header Pattern": { "name": "FlowFile Attribute Header Pattern", "displayName": "FlowFile Attribute Header Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "success", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "ccf5a9db-6d16-3aa5-b054-b33a51405240", "instanceIdentifier": "6304de41-531f-37e9-b1a9-db89a7181c46", "name": "Extract station_uuid", "comments": "", "position": { "x": 1237.9999979753957, "y": 726.9999983799049 }, "type": "org.apache.nifi.processors.standard.EvaluateJsonPath", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Destination": "flowfile-attribute", "Max String Length": "20 MB", "Return Type": "auto-detect", "Null Value Representation": "empty string", "station_uuid": "$.uuid", "Path Not Found Behavior": "ignore" }, "propertyDescriptors": { "Destination": { "name": "Destination", "displayName": "Destination", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Max String Length": { "name": "Max String Length", "displayName": "Max String Length", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Return Type": { "name": "Return Type", "displayName": "Return Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Null Value Representation": { "name": "Null Value Representation", "displayName": "Null Value Representation", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "station_uuid": { "name": "station_uuid", "displayName": "station_uuid", "identifiesControllerService": false, "sensitive": false, "dynamic": true }, "Path Not Found Behavior": { "name": "Path Not Found Behavior", "displayName": "Path Not Found Behavior", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "failure", "unmatched" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "d56a1c61-9006-315a-a15c-41439f14d030", "instanceIdentifier": "d78cc57f-9bd1-35a2-9957-a70d52d0fb69", "name": "SplitRecord", "comments": "", "position": { "x": 861.9999979753956, "y": 510.9999983799049 }, "type": "org.apache.nifi.processors.standard.SplitRecord", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Record Writer": "f6b7ad0f-da95-39c4-be8d-4ae944d8034b", "Record Reader": "e75c34eb-8e60-3b6b-8a29-3dd7b79776c4", "Records Per Split": "1" }, "propertyDescriptors": { "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Records Per Split": { "name": "Records Per Split", "displayName": "Records Per Split", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "original", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "a961da8c-d228-3d19-b6ed-2d48ebe35b05", "instanceIdentifier": "15a87495-c9dc-36db-a3f0-828a63a688f9", "name": "Get 30 days historic data", "comments": "", "position": { "x": 861.9999979753956, "y": 966.9999983799049 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations/${station_uuid}/W/measurements.json?start=P30D", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "false", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "26542ec9-e3e3-380f-ab7a-7fa4b2c46d1e", "instanceIdentifier": "815f05d7-0195-1000-0000-000005d97f79", "name": "PublishKafka", "comments": "", "position": { "x": 864.0, "y": 1408.0 }, "type": "org.apache.nifi.kafka.processors.PublishKafka", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-nar", "version": "2.6.0" }, "properties": { "compression.type": "snappy", "acks": "all", "Message Demarcator": null, "Kafka Key Attribute Encoding": "utf-8", "Record Reader": "e75c34eb-8e60-3b6b-8a29-3dd7b79776c4", "Record Metadata Strategy": "FROM_PROPERTIES", "Transactional ID Prefix": null, "Header Encoding": "UTF-8", "max.request.size": "1 MB", "Topic Name": "measurements", "Kafka Key": null, "partition": null, "Kafka Connection Service": "29032623-5723-3b5d-8983-07534da27fd5", "Publish Strategy": "USE_VALUE", "Record Key Writer": null, "Failure Strategy": "Route to Failure", "partitioner.class": "org.apache.kafka.clients.producer.internals.DefaultPartitioner", "Record Writer": "f6b7ad0f-da95-39c4-be8d-4ae944d8034b", "Transactions Enabled": "false", "Message Key Field": null, "FlowFile Attribute Header Pattern": null }, "propertyDescriptors": { "compression.type": { "name": "compression.type", "displayName": "Compression Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "acks": { "name": "acks", "displayName": "Delivery Guarantee", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Demarcator": { "name": "Message Demarcator", "displayName": "Message Demarcator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key Attribute Encoding": { "name": "Kafka Key Attribute Encoding", "displayName": "Kafka Key Attribute Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Metadata Strategy": { "name": "Record Metadata Strategy", "displayName": "Record Metadata Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transactional ID Prefix": { "name": "Transactional ID Prefix", "displayName": "Transactional ID Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Header Encoding": { "name": "Header Encoding", "displayName": "Header Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.request.size": { "name": "max.request.size", "displayName": "Max Request Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Topic Name": { "name": "Topic Name", "displayName": "Topic Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key": { "name": "Kafka Key", "displayName": "Kafka Key", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partition": { "name": "partition", "displayName": "Partition", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Connection Service": { "name": "Kafka Connection Service", "displayName": "Kafka Connection Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Publish Strategy": { "name": "Publish Strategy", "displayName": "Publish Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Key Writer": { "name": "Record Key Writer", "displayName": "Record Key Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Failure Strategy": { "name": "Failure Strategy", "displayName": "Failure Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partitioner.class": { "name": "partitioner.class", "displayName": "Partitioner Class", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Transactions Enabled": { "name": "Transactions Enabled", "displayName": "Transactions Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Key Field": { "name": "Message Key Field", "displayName": "Message Key Field", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "FlowFile Attribute Header Pattern": { "name": "FlowFile Attribute Header Pattern", "displayName": "FlowFile Attribute Header Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "success", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "4db42754-1203-340d-830d-d662d16d13c8", "instanceIdentifier": "d8795306-ee83-3143-b633-fd70c932add5", "name": "Get station list", "comments": "", "position": { "x": 1237.9999979753957, "y": 246.99999837990495 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations.json", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "false", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "15 m", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "PRIMARY", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "d2841e3c-954c-3f6c-8c2b-0b4f20cea21c", "instanceIdentifier": "a11d09f2-9d6d-390d-947c-397b881e980f", "name": "Get 30 minutes of historic data", "comments": "", "position": { "x": 1237.9999979753957, "y": 966.9999983799049 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations/${station_uuid}/W/measurements.json?start=PT30M", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "false", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "1 s", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "f5aefb8c-ec8c-3895-a221-b1514bf49019", "instanceIdentifier": "815b69b0-0195-1000-ffff-ffff8e27806d", "name": "Produce station records", "comments": "", "position": { "x": 488.0, "y": 512.0 }, "type": "org.apache.nifi.kafka.processors.PublishKafka", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-nar", "version": "2.6.0" }, "properties": { "compression.type": "none", "acks": "all", "Message Demarcator": null, "Kafka Key Attribute Encoding": "utf-8", "Record Reader": "e75c34eb-8e60-3b6b-8a29-3dd7b79776c4", "Record Metadata Strategy": "FROM_PROPERTIES", "Transactional ID Prefix": null, "Header Encoding": "UTF-8", "max.request.size": "1 MB", "Topic Name": "stations", "Kafka Key": null, "partition": null, "Kafka Connection Service": "29032623-5723-3b5d-8983-07534da27fd5", "Publish Strategy": "USE_VALUE", "Record Key Writer": null, "Failure Strategy": "Route to Failure", "partitioner.class": "org.apache.kafka.clients.producer.internals.DefaultPartitioner", "Record Writer": "f6b7ad0f-da95-39c4-be8d-4ae944d8034b", "Transactions Enabled": "false", "Message Key Field": null, "FlowFile Attribute Header Pattern": null }, "propertyDescriptors": { "compression.type": { "name": "compression.type", "displayName": "Compression Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "acks": { "name": "acks", "displayName": "Delivery Guarantee", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Demarcator": { "name": "Message Demarcator", "displayName": "Message Demarcator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key Attribute Encoding": { "name": "Kafka Key Attribute Encoding", "displayName": "Kafka Key Attribute Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Metadata Strategy": { "name": "Record Metadata Strategy", "displayName": "Record Metadata Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transactional ID Prefix": { "name": "Transactional ID Prefix", "displayName": "Transactional ID Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Header Encoding": { "name": "Header Encoding", "displayName": "Header Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.request.size": { "name": "max.request.size", "displayName": "Max Request Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Topic Name": { "name": "Topic Name", "displayName": "Topic Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key": { "name": "Kafka Key", "displayName": "Kafka Key", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partition": { "name": "partition", "displayName": "Partition", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Connection Service": { "name": "Kafka Connection Service", "displayName": "Kafka Connection Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Publish Strategy": { "name": "Publish Strategy", "displayName": "Publish Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Key Writer": { "name": "Record Key Writer", "displayName": "Record Key Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Failure Strategy": { "name": "Failure Strategy", "displayName": "Failure Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partitioner.class": { "name": "partitioner.class", "displayName": "Partitioner Class", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Transactions Enabled": { "name": "Transactions Enabled", "displayName": "Transactions Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Key Field": { "name": "Message Key Field", "displayName": "Message Key Field", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "FlowFile Attribute Header Pattern": { "name": "FlowFile Attribute Header Pattern", "displayName": "FlowFile Attribute Header Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "success", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" } ], "inputPorts": [], "outputPorts": [], "connections": [ { "identifier": "1ed79459-2cd4-30cc-b246-7df8c512d6aa", "instanceIdentifier": "f458062a-4156-340b-99e1-2c9c1fd5691d", "name": "", "source": { "id": "4db42754-1203-340d-830d-d662d16d13c8", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Get station list", "comments": "", "instanceIdentifier": "d8795306-ee83-3143-b633-fd70c932add5" }, "destination": { "id": "656d02d2-3100-39e6-a3dd-a8fe91b6dd06", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "SplitRecord", "comments": "", "instanceIdentifier": "cec0a8ff-73d7-35d1-9f9c-0c5cc0af6db2" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 1, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "6005f6e4-6a3f-379b-92bb-af654e9c13d8", "instanceIdentifier": "1b239318-886a-3f35-a3d4-c7a19aa52cad", "name": "", "source": { "id": "d56a1c61-9006-315a-a15c-41439f14d030", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "SplitRecord", "comments": "", "instanceIdentifier": "d78cc57f-9bd1-35a2-9957-a70d52d0fb69" }, "destination": { "id": "8809ece7-fbe0-3613-b7b0-77e05c8f4cb1", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Extract station_uuid", "comments": "", "instanceIdentifier": "1425717b-a077-39b0-872e-7381d0b3848f" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "splits" ], "backPressureObjectThreshold": 10, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "ROUND_ROBIN", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "be649012-9cd8-3408-aef8-361c82058108", "instanceIdentifier": "8163d1b9-0195-1000-0000-000023899a37", "name": "", "source": { "id": "50bc20ed-2b1b-3068-0000-0000423c0a67", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Add station_uuid", "comments": "", "instanceIdentifier": "3c9c0c1f-db51-39f1-bcea-5aa511e75799" }, "destination": { "id": "98a12d17-c277-3226-af8b-03848dbdbf11", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "PublishKafka", "comments": "", "instanceIdentifier": "816125f7-0195-1000-0000-00006b8c4170" }, "labelIndex": 0, "zIndex": 0, "selectedRelationships": [ "success" ], "backPressureObjectThreshold": 20, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "18447f5e-4e45-39bc-80ac-86ef6083a913", "instanceIdentifier": "40d45abb-c0bb-3d35-a5e9-d4c28528445c", "name": "", "source": { "id": "ccf5a9db-6d16-3aa5-b054-b33a51405240", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Extract station_uuid", "comments": "", "instanceIdentifier": "6304de41-531f-37e9-b1a9-db89a7181c46" }, "destination": { "id": "d2841e3c-954c-3f6c-8c2b-0b4f20cea21c", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Get 30 minutes of historic data", "comments": "", "instanceIdentifier": "a11d09f2-9d6d-390d-947c-397b881e980f" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "matched" ], "backPressureObjectThreshold": 10, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "d7a99464-402e-35fc-b090-3bc3d5fcb68a", "instanceIdentifier": "d5259744-57c7-3711-a0da-8114cb7db873", "name": "", "source": { "id": "8809ece7-fbe0-3613-b7b0-77e05c8f4cb1", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Extract station_uuid", "comments": "", "instanceIdentifier": "1425717b-a077-39b0-872e-7381d0b3848f" }, "destination": { "id": "a961da8c-d228-3d19-b6ed-2d48ebe35b05", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Get 30 days historic data", "comments": "", "instanceIdentifier": "15a87495-c9dc-36db-a3f0-828a63a688f9" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "matched" ], "backPressureObjectThreshold": 10, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "3b89e1cf-29b9-3e53-bcc5-a58dc0db17a9", "instanceIdentifier": "296a4a6e-cf82-39bb-a329-9fc2a74652e7", "name": "", "source": { "id": "e85c2e60-192e-3ed8-ab64-750311e0ed45", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Get station list", "comments": "", "instanceIdentifier": "46c0209c-7abe-3a8e-bbc0-39f9333f14b1" }, "destination": { "id": "f5aefb8c-ec8c-3895-a221-b1514bf49019", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Produce station records", "comments": "", "instanceIdentifier": "815b69b0-0195-1000-ffff-ffff8e27806d" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 10000, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "0085c625-9d46-3f18-8d05-2d55c274a50e", "instanceIdentifier": "50823b83-873f-3bcd-b044-a85de2c53cf9", "name": "", "source": { "id": "656d02d2-3100-39e6-a3dd-a8fe91b6dd06", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "SplitRecord", "comments": "", "instanceIdentifier": "cec0a8ff-73d7-35d1-9f9c-0c5cc0af6db2" }, "destination": { "id": "ccf5a9db-6d16-3aa5-b054-b33a51405240", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Extract station_uuid", "comments": "", "instanceIdentifier": "6304de41-531f-37e9-b1a9-db89a7181c46" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "splits" ], "backPressureObjectThreshold": 10, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "ROUND_ROBIN", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "9424162b-ce44-33ee-8b7a-cfea386e47e4", "instanceIdentifier": "74f2322f-a826-39db-87e3-114ce6a576e4", "name": "", "source": { "id": "e2bdb78b-009b-3086-ad39-816874d3be80", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Get station list", "comments": "", "instanceIdentifier": "72294ac8-46e9-33fb-8923-8e6cb4c69269" }, "destination": { "id": "d56a1c61-9006-315a-a15c-41439f14d030", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "SplitRecord", "comments": "", "instanceIdentifier": "d78cc57f-9bd1-35a2-9957-a70d52d0fb69" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 1, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "776daaa4-a1f6-38a7-b233-2102ddf589d8", "instanceIdentifier": "fbb55c02-33f9-3651-8e7d-f55f264e0514", "name": "", "source": { "id": "d2841e3c-954c-3f6c-8c2b-0b4f20cea21c", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Get 30 minutes of historic data", "comments": "", "instanceIdentifier": "a11d09f2-9d6d-390d-947c-397b881e980f" }, "destination": { "id": "50bc20ed-2b1b-3068-0000-0000423c0a67", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Add station_uuid", "comments": "", "instanceIdentifier": "3c9c0c1f-db51-39f1-bcea-5aa511e75799" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 10, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "82121195-d4b9-37e9-9994-87a6d890cc71", "instanceIdentifier": "8162fee0-0195-1000-ffff-ffff961b93ba", "name": "", "source": { "id": "dde01269-6b44-3f4c-9406-8e291cd8aaa4", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Add station_uuid", "comments": "", "instanceIdentifier": "816243be-0195-1000-ffff-fffff00620a1" }, "destination": { "id": "26542ec9-e3e3-380f-ab7a-7fa4b2c46d1e", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "PublishKafka", "comments": "", "instanceIdentifier": "815f05d7-0195-1000-0000-000005d97f79" }, "labelIndex": 0, "zIndex": 0, "selectedRelationships": [ "success" ], "backPressureObjectThreshold": 20, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "e911351c-a4a5-3a14-9c74-ebd96eef0f8d", "instanceIdentifier": "8b7cc62b-560c-3b8e-903a-f9572ac50df8", "name": "", "source": { "id": "a961da8c-d228-3d19-b6ed-2d48ebe35b05", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Get 30 days historic data", "comments": "", "instanceIdentifier": "15a87495-c9dc-36db-a3f0-828a63a688f9" }, "destination": { "id": "dde01269-6b44-3f4c-9406-8e291cd8aaa4", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Add station_uuid", "comments": "", "instanceIdentifier": "816243be-0195-1000-ffff-fffff00620a1" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 10, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" } ], "labels": [ { "identifier": "10ab1424-3268-3b7f-9e4e-d843df2427cd", "instanceIdentifier": "a57103e7-30b7-38d7-bb6c-f80b4dc4a710", "position": { "x": 485.99999797539556, "y": 174.99999837990495 }, "label": "Water levels data", "zIndex": 0, "width": 1104.0, "height": 24.0, "style": { "font-size": "12px" }, "componentType": "LABEL", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "938fbab6-21ee-347f-814a-b0016b7405b3", "instanceIdentifier": "99926082-4766-3223-8332-3f4affeae76b", "position": { "x": 1237.9999979753957, "y": 214.99999837990495 }, "label": "Stream real-time data", "zIndex": 0, "width": 152.0, "height": 24.0, "style": { "font-size": "12px" }, "componentType": "LABEL", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "b80512f7-0274-3712-8884-d6b31555927a", "instanceIdentifier": "7ef5e570-ca1c-3a70-abf7-9749db121fbc", "position": { "x": 485.99999797539556, "y": 214.99999837990495 }, "label": "Ingest station records", "zIndex": 0, "width": 152.0, "height": 24.0, "style": { "font-size": "12px" }, "componentType": "LABEL", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "4524d905-b8c4-32f6-ae83-4ecc54f95bfa", "instanceIdentifier": "a501f0f6-3372-3444-829c-2109de521069", "position": { "x": 861.9999979753956, "y": 214.99999837990495 }, "label": "Ingest historic data", "zIndex": 0, "width": 152.0, "height": 24.0, "style": { "font-size": "12px" }, "componentType": "LABEL", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" } ], "funnels": [], "controllerServices": [ { "identifier": "a43e5844-29fd-3ad4-8f84-5cfb771da897", "instanceIdentifier": "3213e3ae-00c3-3dd0-a838-770a15dc68e0", "name": "StandardRestrictedSSLContextService", "comments": "", "type": "org.apache.nifi.ssl.StandardRestrictedSSLContextService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-ssl-context-service-nar", "version": "2.6.0" }, "properties": { "Truststore Type": "PKCS12", "Truststore Password": "secret", "SSL Protocol": "TLSv1.3", "Keystore Type": "PKCS12", "Keystore Password": "secret", "Truststore Filename": "/stackable/keystore/truststore.p12", "Keystore Filename": "/stackable/keystore/keystore.p12" }, "propertyDescriptors": { "Truststore Type": { "name": "Truststore Type", "displayName": "Truststore Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Protocol": { "name": "SSL Protocol", "displayName": "TLS Protocol", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Keystore Type": { "name": "Keystore Type", "displayName": "Keystore Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Truststore Filename": { "name": "Truststore Filename", "displayName": "Truststore Filename", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "SINGLE", "resourceTypes": [ "FILE" ] } }, "Keystore Password": { "name": "Keystore Password", "displayName": "Keystore Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "key-password": { "name": "key-password", "displayName": "Key Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Truststore Password": { "name": "Truststore Password", "displayName": "Truststore Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Keystore Filename": { "name": "Keystore Filename", "displayName": "Keystore Filename", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "SINGLE", "resourceTypes": [ "FILE" ] } } }, "controllerServiceApis": [ { "type": "org.apache.nifi.ssl.SSLContextProvider", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } }, { "type": "org.apache.nifi.ssl.SSLContextService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } }, { "type": "org.apache.nifi.ssl.RestrictedSSLContextService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } } ], "scheduledState": "DISABLED", "bulletinLevel": "WARN", "componentType": "CONTROLLER_SERVICE", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "f6b7ad0f-da95-39c4-be8d-4ae944d8034b", "instanceIdentifier": "ca38b3a6-868b-33f0-a858-30da336cc679", "name": "JsonRecordSetWriter", "type": "org.apache.nifi.json.JsonRecordSetWriter", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-record-serialization-services-nar", "version": "2.6.0" }, "properties": { "schema-reference-reader": null, "schema-branch": null, "Allow Scientific Notation": "false", "compression-level": "1", "schema-cache": null, "Timestamp Format": null, "Date Format": null, "Pretty Print JSON": "false", "compression-format": "none", "Schema Write Strategy": "no-schema", "suppress-nulls": "never-suppress", "output-grouping": "output-array", "schema-name": "${schema.name}", "schema-registry": null, "Time Format": null, "schema-access-strategy": "inherit-record-schema", "schema-version": null, "schema-text": "${avro.schema}", "Schema Reference Writer": null }, "propertyDescriptors": { "schema-reference-reader": { "name": "schema-reference-reader", "displayName": "Schema Reference Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "schema-branch": { "name": "schema-branch", "displayName": "Schema Branch", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Allow Scientific Notation": { "name": "Allow Scientific Notation", "displayName": "Allow Scientific Notation", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "compression-level": { "name": "compression-level", "displayName": "Compression Level", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-cache": { "name": "schema-cache", "displayName": "Schema Cache", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Timestamp Format": { "name": "Timestamp Format", "displayName": "Timestamp Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Date Format": { "name": "Date Format", "displayName": "Date Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Pretty Print JSON": { "name": "Pretty Print JSON", "displayName": "Pretty Print JSON", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "compression-format": { "name": "compression-format", "displayName": "Compression Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Schema Write Strategy": { "name": "Schema Write Strategy", "displayName": "Schema Write Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "suppress-nulls": { "name": "suppress-nulls", "displayName": "Suppress Null Values", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "output-grouping": { "name": "output-grouping", "displayName": "Output Grouping", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-name": { "name": "schema-name", "displayName": "Schema Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-registry": { "name": "schema-registry", "displayName": "Schema Registry", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Time Format": { "name": "Time Format", "displayName": "Time Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-access-strategy": { "name": "schema-access-strategy", "displayName": "Schema Access Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-version": { "name": "schema-version", "displayName": "Schema Version", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-text": { "name": "schema-text", "displayName": "Schema Text", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Schema Reference Writer": { "name": "Schema Reference Writer", "displayName": "Schema Reference Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false } }, "controllerServiceApis": [ { "type": "org.apache.nifi.serialization.RecordSetWriterFactory", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } } ], "scheduledState": "DISABLED", "bulletinLevel": "WARN", "componentType": "CONTROLLER_SERVICE", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "e75c34eb-8e60-3b6b-8a29-3dd7b79776c4", "instanceIdentifier": "2d152882-2f62-3e45-8c8e-5f3127f8b931", "name": "JsonTreeReader", "type": "org.apache.nifi.json.JsonTreeReader", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-record-serialization-services-nar", "version": "2.6.0" }, "properties": { "schema-reference-reader": null, "schema-branch": null, "Max String Length": "20 MB", "schema-application-strategy": "SELECTED_PART", "Timestamp Format": null, "schema-inference-cache": null, "Date Format": null, "schema-name": "${schema.name}", "starting-field-strategy": "ROOT_NODE", "schema-registry": null, "starting-field-name": null, "Time Format": null, "schema-access-strategy": "infer-schema", "schema-version": null, "schema-text": "${avro.schema}", "Allow Comments": "false" }, "propertyDescriptors": { "schema-reference-reader": { "name": "schema-reference-reader", "displayName": "Schema Reference Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "schema-branch": { "name": "schema-branch", "displayName": "Schema Branch", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Max String Length": { "name": "Max String Length", "displayName": "Max String Length", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-application-strategy": { "name": "schema-application-strategy", "displayName": "Schema Application Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Timestamp Format": { "name": "Timestamp Format", "displayName": "Timestamp Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-inference-cache": { "name": "schema-inference-cache", "displayName": "Schema Inference Cache", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Date Format": { "name": "Date Format", "displayName": "Date Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-name": { "name": "schema-name", "displayName": "Schema Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "starting-field-strategy": { "name": "starting-field-strategy", "displayName": "Starting Field Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-registry": { "name": "schema-registry", "displayName": "Schema Registry", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "starting-field-name": { "name": "starting-field-name", "displayName": "Starting Field Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Time Format": { "name": "Time Format", "displayName": "Time Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-access-strategy": { "name": "schema-access-strategy", "displayName": "Schema Access Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-version": { "name": "schema-version", "displayName": "Schema Version", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-text": { "name": "schema-text", "displayName": "Schema Text", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Allow Comments": { "name": "Allow Comments", "displayName": "Allow Comments", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "controllerServiceApis": [ { "type": "org.apache.nifi.serialization.RecordReaderFactory", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } } ], "scheduledState": "DISABLED", "bulletinLevel": "WARN", "componentType": "CONTROLLER_SERVICE", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "29032623-5723-3b5d-8983-07534da27fd5", "instanceIdentifier": "815ba7c0-0195-1000-0000-00002ff4760b", "name": "Kafka3ConnectionService", "comments": "", "type": "org.apache.nifi.kafka.service.Kafka3ConnectionService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-3-service-nar", "version": "2.6.0" }, "properties": { "security.protocol": "SSL", "SSL Context Service": "a43e5844-29fd-3ad4-8f84-5cfb771da897", "ack.wait.time": "5 sec", "sasl.username": null, "max.poll.records": "10000", "sasl.mechanism": "GSSAPI", "isolation.level": "read_committed", "max.block.ms": "5 sec", "bootstrap.servers": "kafka-broker-default-headless.default.svc.cluster.local:9093", "kerberos-user-service": null, "sasl.kerberos.service.name": null, "default.api.timeout.ms": "60 sec" }, "propertyDescriptors": { "ack.wait.time": { "name": "ack.wait.time", "displayName": "Acknowledgment Wait Time", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "sasl.username": { "name": "sasl.username", "displayName": "SASL Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.poll.records": { "name": "max.poll.records", "displayName": "Max Poll Records", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "bootstrap.servers": { "name": "bootstrap.servers", "displayName": "Bootstrap Servers", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "sasl.kerberos.service.name": { "name": "sasl.kerberos.service.name", "displayName": "Kerberos Service Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "security.protocol": { "name": "security.protocol", "displayName": "Security Protocol", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "sasl.mechanism": { "name": "sasl.mechanism", "displayName": "SASL Mechanism", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "isolation.level": { "name": "isolation.level", "displayName": "Transaction Isolation Level", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.block.ms": { "name": "max.block.ms", "displayName": "Max Metadata Wait Time", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "kerberos-user-service": { "name": "kerberos-user-service", "displayName": "Kerberos User Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "sasl.password": { "name": "sasl.password", "displayName": "SASL Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "default.api.timeout.ms": { "name": "default.api.timeout.ms", "displayName": "Client Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "controllerServiceApis": [ { "type": "org.apache.nifi.kafka.service.api.KafkaConnectionService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-service-api-nar", "version": "2.6.0" } } ], "scheduledState": "DISABLED", "bulletinLevel": "WARN", "componentType": "CONTROLLER_SERVICE", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" } ], "defaultFlowFileExpiration": "0 sec", "defaultBackPressureObjectThreshold": 10000, "defaultBackPressureDataSizeThreshold": "1 GB", "scheduledState": "ENABLED", "executionEngine": "INHERITED", "maxConcurrentTasks": 1, "statelessFlowTimeout": "1 min", "flowFileConcurrency": "UNBOUNDED", "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE", "componentType": "PROCESS_GROUP" }, "externalControllerServices": {}, "parameterContexts": {}, "flowEncodingVersion": "1.0", "parameterProviders": {}, "latest": false } + { "flowContents": { "identifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "instanceIdentifier": "81598f35-0195-1000-0000-000073723729", "name": "IngestWaterLevelsToKafka_raw", "comments": "", "position": { "x": -472.0, "y": -496.0 }, "processGroups": [], "remoteProcessGroups": [], "processors": [ { "identifier": "e85c2e60-192e-3ed8-ab64-750311e0ed45", "instanceIdentifier": "46c0209c-7abe-3a8e-bbc0-39f9333f14b1", "name": "Get station list", "comments": "", "position": { "x": 485.99999797539556, "y": 246.99999837990495 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations.json", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "false", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "365000 days", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "PRIMARY", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "50bc20ed-2b1b-3068-0000-0000423c0a67", "instanceIdentifier": "3c9c0c1f-db51-39f1-bcea-5aa511e75799", "name": "Add station_uuid", "comments": "", "position": { "x": 1240.0, "y": 1192.0 }, "type": "org.apache.nifi.processors.jolt.JoltTransformJSON", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-jolt-nar", "version": "2.6.0" }, "properties": { "Jolt Transform": "jolt-transform-chain", "Max String Length": "20 MB", "Transform Cache Size": "1", "Custom Transformation Class Name": null, "Custom Module Directory": null, "Pretty Print": "false", "Jolt Specification": "[\n {\n \"operation\": \"default\",\n \"spec\": {\n \"*\": {\n \"station_uuid\": \"${station_uuid}\"\n }\n }\n }\n]\n" }, "propertyDescriptors": { "Jolt Transform": { "name": "Jolt Transform", "displayName": "Jolt Transform", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Max String Length": { "name": "Max String Length", "displayName": "Max String Length", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transform Cache Size": { "name": "Transform Cache Size", "displayName": "Transform Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Transformation Class Name": { "name": "Custom Transformation Class Name", "displayName": "Custom Transformation Class Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Module Directory": { "name": "Custom Module Directory", "displayName": "Custom Module Directory", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "MULTIPLE", "resourceTypes": [ "DIRECTORY", "FILE" ] } }, "Pretty Print": { "name": "Pretty Print", "displayName": "Pretty Print", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Jolt Specification": { "name": "Jolt Specification", "displayName": "Jolt Specification", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "SINGLE", "resourceTypes": [ "TEXT", "FILE" ] } } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "8809ece7-fbe0-3613-b7b0-77e05c8f4cb1", "instanceIdentifier": "1425717b-a077-39b0-872e-7381d0b3848f", "name": "Extract station_uuid", "comments": "", "position": { "x": 861.9999979753956, "y": 726.9999983799049 }, "type": "org.apache.nifi.processors.standard.EvaluateJsonPath", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Destination": "flowfile-attribute", "Max String Length": "20 MB", "Return Type": "auto-detect", "Null Value Representation": "empty string", "station_uuid": "$.uuid", "Path Not Found Behavior": "ignore" }, "propertyDescriptors": { "Destination": { "name": "Destination", "displayName": "Destination", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Max String Length": { "name": "Max String Length", "displayName": "Max String Length", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Return Type": { "name": "Return Type", "displayName": "Return Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Null Value Representation": { "name": "Null Value Representation", "displayName": "Null Value Representation", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "station_uuid": { "name": "station_uuid", "displayName": "station_uuid", "identifiesControllerService": false, "sensitive": false, "dynamic": true }, "Path Not Found Behavior": { "name": "Path Not Found Behavior", "displayName": "Path Not Found Behavior", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "failure", "unmatched" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "656d02d2-3100-39e6-a3dd-a8fe91b6dd06", "instanceIdentifier": "cec0a8ff-73d7-35d1-9f9c-0c5cc0af6db2", "name": "SplitRecord", "comments": "", "position": { "x": 1237.9999979753957, "y": 510.9999983799049 }, "type": "org.apache.nifi.processors.standard.SplitRecord", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Record Writer": "f6b7ad0f-da95-39c4-be8d-4ae944d8034b", "Record Reader": "e75c34eb-8e60-3b6b-8a29-3dd7b79776c4", "Records Per Split": "1" }, "propertyDescriptors": { "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Records Per Split": { "name": "Records Per Split", "displayName": "Records Per Split", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "original", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "e2bdb78b-009b-3086-ad39-816874d3be80", "instanceIdentifier": "72294ac8-46e9-33fb-8923-8e6cb4c69269", "name": "Get station list", "comments": "", "position": { "x": 861.9999979753956, "y": 246.99999837990495 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations.json", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "false", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "365000 days", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "PRIMARY", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "dde01269-6b44-3f4c-9406-8e291cd8aaa4", "instanceIdentifier": "816243be-0195-1000-ffff-fffff00620a1", "name": "Add station_uuid", "comments": "", "position": { "x": 864.0, "y": 1192.0 }, "type": "org.apache.nifi.processors.jolt.JoltTransformJSON", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-jolt-nar", "version": "2.6.0" }, "properties": { "Jolt Transform": "jolt-transform-chain", "Max String Length": "20 MB", "Transform Cache Size": "1", "Custom Transformation Class Name": null, "Custom Module Directory": null, "Pretty Print": "false", "Jolt Specification": "[\n {\n \"operation\": \"default\",\n \"spec\": {\n \"*\": {\n \"station_uuid\": \"${station_uuid}\"\n }\n }\n }\n]\n" }, "propertyDescriptors": { "Jolt Transform": { "name": "Jolt Transform", "displayName": "Jolt Transform", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Max String Length": { "name": "Max String Length", "displayName": "Max String Length", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transform Cache Size": { "name": "Transform Cache Size", "displayName": "Transform Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Transformation Class Name": { "name": "Custom Transformation Class Name", "displayName": "Custom Transformation Class Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Module Directory": { "name": "Custom Module Directory", "displayName": "Custom Module Directory", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "MULTIPLE", "resourceTypes": [ "DIRECTORY", "FILE" ] } }, "Pretty Print": { "name": "Pretty Print", "displayName": "Pretty Print", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Jolt Specification": { "name": "Jolt Specification", "displayName": "Jolt Specification", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "SINGLE", "resourceTypes": [ "TEXT", "FILE" ] } } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "98a12d17-c277-3226-af8b-03848dbdbf11", "instanceIdentifier": "816125f7-0195-1000-0000-00006b8c4170", "name": "PublishKafka", "comments": "", "position": { "x": 1240.0, "y": 1408.0 }, "type": "org.apache.nifi.kafka.processors.PublishKafka", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-nar", "version": "2.6.0" }, "properties": { "compression.type": "snappy", "acks": "all", "Message Demarcator": null, "Kafka Key Attribute Encoding": "utf-8", "Record Reader": "e75c34eb-8e60-3b6b-8a29-3dd7b79776c4", "Record Metadata Strategy": "FROM_PROPERTIES", "Transactional ID Prefix": null, "Header Encoding": "UTF-8", "max.request.size": "1 MB", "Topic Name": "measurements", "Kafka Key": null, "partition": null, "Kafka Connection Service": "29032623-5723-3b5d-8983-07534da27fd5", "Publish Strategy": "USE_VALUE", "Record Key Writer": null, "Failure Strategy": "Route to Failure", "partitioner.class": "org.apache.kafka.clients.producer.internals.DefaultPartitioner", "Record Writer": "f6b7ad0f-da95-39c4-be8d-4ae944d8034b", "Transactions Enabled": "false", "Message Key Field": null, "FlowFile Attribute Header Pattern": null }, "propertyDescriptors": { "compression.type": { "name": "compression.type", "displayName": "Compression Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "acks": { "name": "acks", "displayName": "Delivery Guarantee", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Demarcator": { "name": "Message Demarcator", "displayName": "Message Demarcator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key Attribute Encoding": { "name": "Kafka Key Attribute Encoding", "displayName": "Kafka Key Attribute Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Metadata Strategy": { "name": "Record Metadata Strategy", "displayName": "Record Metadata Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transactional ID Prefix": { "name": "Transactional ID Prefix", "displayName": "Transactional ID Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Header Encoding": { "name": "Header Encoding", "displayName": "Header Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.request.size": { "name": "max.request.size", "displayName": "Max Request Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Topic Name": { "name": "Topic Name", "displayName": "Topic Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key": { "name": "Kafka Key", "displayName": "Kafka Key", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partition": { "name": "partition", "displayName": "Partition", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Connection Service": { "name": "Kafka Connection Service", "displayName": "Kafka Connection Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Publish Strategy": { "name": "Publish Strategy", "displayName": "Publish Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Key Writer": { "name": "Record Key Writer", "displayName": "Record Key Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Failure Strategy": { "name": "Failure Strategy", "displayName": "Failure Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partitioner.class": { "name": "partitioner.class", "displayName": "Partitioner Class", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Transactions Enabled": { "name": "Transactions Enabled", "displayName": "Transactions Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Key Field": { "name": "Message Key Field", "displayName": "Message Key Field", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "FlowFile Attribute Header Pattern": { "name": "FlowFile Attribute Header Pattern", "displayName": "FlowFile Attribute Header Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "success", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "ccf5a9db-6d16-3aa5-b054-b33a51405240", "instanceIdentifier": "6304de41-531f-37e9-b1a9-db89a7181c46", "name": "Extract station_uuid", "comments": "", "position": { "x": 1237.9999979753957, "y": 726.9999983799049 }, "type": "org.apache.nifi.processors.standard.EvaluateJsonPath", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Destination": "flowfile-attribute", "Max String Length": "20 MB", "Return Type": "auto-detect", "Null Value Representation": "empty string", "station_uuid": "$.uuid", "Path Not Found Behavior": "ignore" }, "propertyDescriptors": { "Destination": { "name": "Destination", "displayName": "Destination", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Max String Length": { "name": "Max String Length", "displayName": "Max String Length", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Return Type": { "name": "Return Type", "displayName": "Return Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Null Value Representation": { "name": "Null Value Representation", "displayName": "Null Value Representation", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "station_uuid": { "name": "station_uuid", "displayName": "station_uuid", "identifiesControllerService": false, "sensitive": false, "dynamic": true }, "Path Not Found Behavior": { "name": "Path Not Found Behavior", "displayName": "Path Not Found Behavior", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "failure", "unmatched" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "d56a1c61-9006-315a-a15c-41439f14d030", "instanceIdentifier": "d78cc57f-9bd1-35a2-9957-a70d52d0fb69", "name": "SplitRecord", "comments": "", "position": { "x": 861.9999979753956, "y": 510.9999983799049 }, "type": "org.apache.nifi.processors.standard.SplitRecord", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Record Writer": "f6b7ad0f-da95-39c4-be8d-4ae944d8034b", "Record Reader": "e75c34eb-8e60-3b6b-8a29-3dd7b79776c4", "Records Per Split": "1" }, "propertyDescriptors": { "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Records Per Split": { "name": "Records Per Split", "displayName": "Records Per Split", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "original", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "a961da8c-d228-3d19-b6ed-2d48ebe35b05", "instanceIdentifier": "15a87495-c9dc-36db-a3f0-828a63a688f9", "name": "Get 30 days historic data", "comments": "", "position": { "x": 861.9999979753956, "y": 966.9999983799049 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations/${station_uuid}/W/measurements.json?start=P30D", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "false", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "26542ec9-e3e3-380f-ab7a-7fa4b2c46d1e", "instanceIdentifier": "815f05d7-0195-1000-0000-000005d97f79", "name": "PublishKafka", "comments": "", "position": { "x": 864.0, "y": 1408.0 }, "type": "org.apache.nifi.kafka.processors.PublishKafka", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-nar", "version": "2.6.0" }, "properties": { "compression.type": "snappy", "acks": "all", "Message Demarcator": null, "Kafka Key Attribute Encoding": "utf-8", "Record Reader": "e75c34eb-8e60-3b6b-8a29-3dd7b79776c4", "Record Metadata Strategy": "FROM_PROPERTIES", "Transactional ID Prefix": null, "Header Encoding": "UTF-8", "max.request.size": "1 MB", "Topic Name": "measurements", "Kafka Key": null, "partition": null, "Kafka Connection Service": "29032623-5723-3b5d-8983-07534da27fd5", "Publish Strategy": "USE_VALUE", "Record Key Writer": null, "Failure Strategy": "Route to Failure", "partitioner.class": "org.apache.kafka.clients.producer.internals.DefaultPartitioner", "Record Writer": "f6b7ad0f-da95-39c4-be8d-4ae944d8034b", "Transactions Enabled": "false", "Message Key Field": null, "FlowFile Attribute Header Pattern": null }, "propertyDescriptors": { "compression.type": { "name": "compression.type", "displayName": "Compression Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "acks": { "name": "acks", "displayName": "Delivery Guarantee", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Demarcator": { "name": "Message Demarcator", "displayName": "Message Demarcator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key Attribute Encoding": { "name": "Kafka Key Attribute Encoding", "displayName": "Kafka Key Attribute Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Metadata Strategy": { "name": "Record Metadata Strategy", "displayName": "Record Metadata Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transactional ID Prefix": { "name": "Transactional ID Prefix", "displayName": "Transactional ID Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Header Encoding": { "name": "Header Encoding", "displayName": "Header Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.request.size": { "name": "max.request.size", "displayName": "Max Request Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Topic Name": { "name": "Topic Name", "displayName": "Topic Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key": { "name": "Kafka Key", "displayName": "Kafka Key", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partition": { "name": "partition", "displayName": "Partition", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Connection Service": { "name": "Kafka Connection Service", "displayName": "Kafka Connection Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Publish Strategy": { "name": "Publish Strategy", "displayName": "Publish Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Key Writer": { "name": "Record Key Writer", "displayName": "Record Key Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Failure Strategy": { "name": "Failure Strategy", "displayName": "Failure Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partitioner.class": { "name": "partitioner.class", "displayName": "Partitioner Class", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Transactions Enabled": { "name": "Transactions Enabled", "displayName": "Transactions Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Key Field": { "name": "Message Key Field", "displayName": "Message Key Field", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "FlowFile Attribute Header Pattern": { "name": "FlowFile Attribute Header Pattern", "displayName": "FlowFile Attribute Header Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "success", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "4db42754-1203-340d-830d-d662d16d13c8", "instanceIdentifier": "d8795306-ee83-3143-b633-fd70c932add5", "name": "Get station list", "comments": "", "position": { "x": 1237.9999979753957, "y": 246.99999837990495 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations.json", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "false", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "15 m", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "PRIMARY", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "d2841e3c-954c-3f6c-8c2b-0b4f20cea21c", "instanceIdentifier": "a11d09f2-9d6d-390d-947c-397b881e980f", "name": "Get 30 minutes of historic data", "comments": "", "position": { "x": 1237.9999979753957, "y": 966.9999983799049 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations/${station_uuid}/W/measurements.json?start=PT30M", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "false", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "1 s", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "f5aefb8c-ec8c-3895-a221-b1514bf49019", "instanceIdentifier": "815b69b0-0195-1000-ffff-ffff8e27806d", "name": "Produce station records", "comments": "", "position": { "x": 488.0, "y": 512.0 }, "type": "org.apache.nifi.kafka.processors.PublishKafka", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-nar", "version": "2.6.0" }, "properties": { "compression.type": "none", "acks": "all", "Message Demarcator": null, "Kafka Key Attribute Encoding": "utf-8", "Record Reader": "e75c34eb-8e60-3b6b-8a29-3dd7b79776c4", "Record Metadata Strategy": "FROM_PROPERTIES", "Transactional ID Prefix": null, "Header Encoding": "UTF-8", "max.request.size": "1 MB", "Topic Name": "stations", "Kafka Key": null, "partition": null, "Kafka Connection Service": "29032623-5723-3b5d-8983-07534da27fd5", "Publish Strategy": "USE_VALUE", "Record Key Writer": null, "Failure Strategy": "Route to Failure", "partitioner.class": "org.apache.kafka.clients.producer.internals.DefaultPartitioner", "Record Writer": "f6b7ad0f-da95-39c4-be8d-4ae944d8034b", "Transactions Enabled": "false", "Message Key Field": null, "FlowFile Attribute Header Pattern": null }, "propertyDescriptors": { "compression.type": { "name": "compression.type", "displayName": "Compression Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "acks": { "name": "acks", "displayName": "Delivery Guarantee", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Demarcator": { "name": "Message Demarcator", "displayName": "Message Demarcator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key Attribute Encoding": { "name": "Kafka Key Attribute Encoding", "displayName": "Kafka Key Attribute Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Metadata Strategy": { "name": "Record Metadata Strategy", "displayName": "Record Metadata Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transactional ID Prefix": { "name": "Transactional ID Prefix", "displayName": "Transactional ID Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Header Encoding": { "name": "Header Encoding", "displayName": "Header Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.request.size": { "name": "max.request.size", "displayName": "Max Request Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Topic Name": { "name": "Topic Name", "displayName": "Topic Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key": { "name": "Kafka Key", "displayName": "Kafka Key", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partition": { "name": "partition", "displayName": "Partition", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Connection Service": { "name": "Kafka Connection Service", "displayName": "Kafka Connection Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Publish Strategy": { "name": "Publish Strategy", "displayName": "Publish Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Key Writer": { "name": "Record Key Writer", "displayName": "Record Key Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Failure Strategy": { "name": "Failure Strategy", "displayName": "Failure Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partitioner.class": { "name": "partitioner.class", "displayName": "Partitioner Class", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Transactions Enabled": { "name": "Transactions Enabled", "displayName": "Transactions Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Key Field": { "name": "Message Key Field", "displayName": "Message Key Field", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "FlowFile Attribute Header Pattern": { "name": "FlowFile Attribute Header Pattern", "displayName": "FlowFile Attribute Header Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "success", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" } ], "inputPorts": [], "outputPorts": [], "connections": [ { "identifier": "1ed79459-2cd4-30cc-b246-7df8c512d6aa", "instanceIdentifier": "f458062a-4156-340b-99e1-2c9c1fd5691d", "name": "", "source": { "id": "4db42754-1203-340d-830d-d662d16d13c8", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Get station list", "comments": "", "instanceIdentifier": "d8795306-ee83-3143-b633-fd70c932add5" }, "destination": { "id": "656d02d2-3100-39e6-a3dd-a8fe91b6dd06", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "SplitRecord", "comments": "", "instanceIdentifier": "cec0a8ff-73d7-35d1-9f9c-0c5cc0af6db2" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 1, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "6005f6e4-6a3f-379b-92bb-af654e9c13d8", "instanceIdentifier": "1b239318-886a-3f35-a3d4-c7a19aa52cad", "name": "", "source": { "id": "d56a1c61-9006-315a-a15c-41439f14d030", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "SplitRecord", "comments": "", "instanceIdentifier": "d78cc57f-9bd1-35a2-9957-a70d52d0fb69" }, "destination": { "id": "8809ece7-fbe0-3613-b7b0-77e05c8f4cb1", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Extract station_uuid", "comments": "", "instanceIdentifier": "1425717b-a077-39b0-872e-7381d0b3848f" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "splits" ], "backPressureObjectThreshold": 10, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "ROUND_ROBIN", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "be649012-9cd8-3408-aef8-361c82058108", "instanceIdentifier": "8163d1b9-0195-1000-0000-000023899a37", "name": "", "source": { "id": "50bc20ed-2b1b-3068-0000-0000423c0a67", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Add station_uuid", "comments": "", "instanceIdentifier": "3c9c0c1f-db51-39f1-bcea-5aa511e75799" }, "destination": { "id": "98a12d17-c277-3226-af8b-03848dbdbf11", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "PublishKafka", "comments": "", "instanceIdentifier": "816125f7-0195-1000-0000-00006b8c4170" }, "labelIndex": 0, "zIndex": 0, "selectedRelationships": [ "success" ], "backPressureObjectThreshold": 20, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "18447f5e-4e45-39bc-80ac-86ef6083a913", "instanceIdentifier": "40d45abb-c0bb-3d35-a5e9-d4c28528445c", "name": "", "source": { "id": "ccf5a9db-6d16-3aa5-b054-b33a51405240", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Extract station_uuid", "comments": "", "instanceIdentifier": "6304de41-531f-37e9-b1a9-db89a7181c46" }, "destination": { "id": "d2841e3c-954c-3f6c-8c2b-0b4f20cea21c", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Get 30 minutes of historic data", "comments": "", "instanceIdentifier": "a11d09f2-9d6d-390d-947c-397b881e980f" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "matched" ], "backPressureObjectThreshold": 10, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "d7a99464-402e-35fc-b090-3bc3d5fcb68a", "instanceIdentifier": "d5259744-57c7-3711-a0da-8114cb7db873", "name": "", "source": { "id": "8809ece7-fbe0-3613-b7b0-77e05c8f4cb1", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Extract station_uuid", "comments": "", "instanceIdentifier": "1425717b-a077-39b0-872e-7381d0b3848f" }, "destination": { "id": "a961da8c-d228-3d19-b6ed-2d48ebe35b05", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Get 30 days historic data", "comments": "", "instanceIdentifier": "15a87495-c9dc-36db-a3f0-828a63a688f9" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "matched" ], "backPressureObjectThreshold": 10, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "3b89e1cf-29b9-3e53-bcc5-a58dc0db17a9", "instanceIdentifier": "296a4a6e-cf82-39bb-a329-9fc2a74652e7", "name": "", "source": { "id": "e85c2e60-192e-3ed8-ab64-750311e0ed45", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Get station list", "comments": "", "instanceIdentifier": "46c0209c-7abe-3a8e-bbc0-39f9333f14b1" }, "destination": { "id": "f5aefb8c-ec8c-3895-a221-b1514bf49019", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Produce station records", "comments": "", "instanceIdentifier": "815b69b0-0195-1000-ffff-ffff8e27806d" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 10000, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "0085c625-9d46-3f18-8d05-2d55c274a50e", "instanceIdentifier": "50823b83-873f-3bcd-b044-a85de2c53cf9", "name": "", "source": { "id": "656d02d2-3100-39e6-a3dd-a8fe91b6dd06", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "SplitRecord", "comments": "", "instanceIdentifier": "cec0a8ff-73d7-35d1-9f9c-0c5cc0af6db2" }, "destination": { "id": "ccf5a9db-6d16-3aa5-b054-b33a51405240", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Extract station_uuid", "comments": "", "instanceIdentifier": "6304de41-531f-37e9-b1a9-db89a7181c46" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "splits" ], "backPressureObjectThreshold": 10, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "ROUND_ROBIN", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "9424162b-ce44-33ee-8b7a-cfea386e47e4", "instanceIdentifier": "74f2322f-a826-39db-87e3-114ce6a576e4", "name": "", "source": { "id": "e2bdb78b-009b-3086-ad39-816874d3be80", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Get station list", "comments": "", "instanceIdentifier": "72294ac8-46e9-33fb-8923-8e6cb4c69269" }, "destination": { "id": "d56a1c61-9006-315a-a15c-41439f14d030", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "SplitRecord", "comments": "", "instanceIdentifier": "d78cc57f-9bd1-35a2-9957-a70d52d0fb69" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 1, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "776daaa4-a1f6-38a7-b233-2102ddf589d8", "instanceIdentifier": "fbb55c02-33f9-3651-8e7d-f55f264e0514", "name": "", "source": { "id": "d2841e3c-954c-3f6c-8c2b-0b4f20cea21c", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Get 30 minutes of historic data", "comments": "", "instanceIdentifier": "a11d09f2-9d6d-390d-947c-397b881e980f" }, "destination": { "id": "50bc20ed-2b1b-3068-0000-0000423c0a67", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Add station_uuid", "comments": "", "instanceIdentifier": "3c9c0c1f-db51-39f1-bcea-5aa511e75799" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 10, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "82121195-d4b9-37e9-9994-87a6d890cc71", "instanceIdentifier": "8162fee0-0195-1000-ffff-ffff961b93ba", "name": "", "source": { "id": "dde01269-6b44-3f4c-9406-8e291cd8aaa4", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Add station_uuid", "comments": "", "instanceIdentifier": "816243be-0195-1000-ffff-fffff00620a1" }, "destination": { "id": "26542ec9-e3e3-380f-ab7a-7fa4b2c46d1e", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "PublishKafka", "comments": "", "instanceIdentifier": "815f05d7-0195-1000-0000-000005d97f79" }, "labelIndex": 0, "zIndex": 0, "selectedRelationships": [ "success" ], "backPressureObjectThreshold": 20, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "e911351c-a4a5-3a14-9c74-ebd96eef0f8d", "instanceIdentifier": "8b7cc62b-560c-3b8e-903a-f9572ac50df8", "name": "", "source": { "id": "a961da8c-d228-3d19-b6ed-2d48ebe35b05", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Get 30 days historic data", "comments": "", "instanceIdentifier": "15a87495-c9dc-36db-a3f0-828a63a688f9" }, "destination": { "id": "dde01269-6b44-3f4c-9406-8e291cd8aaa4", "type": "PROCESSOR", "groupId": "5bb5c008-b432-3d6f-b99e-fedc455bd418", "name": "Add station_uuid", "comments": "", "instanceIdentifier": "816243be-0195-1000-ffff-fffff00620a1" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 10, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" } ], "labels": [ { "identifier": "10ab1424-3268-3b7f-9e4e-d843df2427cd", "instanceIdentifier": "a57103e7-30b7-38d7-bb6c-f80b4dc4a710", "position": { "x": 485.99999797539556, "y": 174.99999837990495 }, "label": "Water levels data", "zIndex": 0, "width": 1104.0, "height": 24.0, "style": { "font-size": "12px" }, "componentType": "LABEL", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "938fbab6-21ee-347f-814a-b0016b7405b3", "instanceIdentifier": "99926082-4766-3223-8332-3f4affeae76b", "position": { "x": 1237.9999979753957, "y": 214.99999837990495 }, "label": "Stream real-time data", "zIndex": 0, "width": 152.0, "height": 24.0, "style": { "font-size": "12px" }, "componentType": "LABEL", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "b80512f7-0274-3712-8884-d6b31555927a", "instanceIdentifier": "7ef5e570-ca1c-3a70-abf7-9749db121fbc", "position": { "x": 485.99999797539556, "y": 214.99999837990495 }, "label": "Ingest station records", "zIndex": 0, "width": 152.0, "height": 24.0, "style": { "font-size": "12px" }, "componentType": "LABEL", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "4524d905-b8c4-32f6-ae83-4ecc54f95bfa", "instanceIdentifier": "a501f0f6-3372-3444-829c-2109de521069", "position": { "x": 861.9999979753956, "y": 214.99999837990495 }, "label": "Ingest historic data", "zIndex": 0, "width": 152.0, "height": 24.0, "style": { "font-size": "12px" }, "componentType": "LABEL", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" } ], "funnels": [], "controllerServices": [ { "identifier": "a43e5844-29fd-3ad4-8f84-5cfb771da897", "instanceIdentifier": "3213e3ae-00c3-3dd0-a838-770a15dc68e0", "name": "StandardRestrictedSSLContextService", "comments": "", "type": "org.apache.nifi.ssl.StandardRestrictedSSLContextService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-ssl-context-service-nar", "version": "2.6.0" }, "properties": { "Truststore Type": "PKCS12", "Truststore Password": "secret", "SSL Protocol": "TLSv1.3", "Keystore Type": "PKCS12", "Keystore Password": "secret", "Truststore Filename": "/stackable/keystore/truststore.p12", "Keystore Filename": "/stackable/keystore/keystore.p12" }, "propertyDescriptors": { "Truststore Type": { "name": "Truststore Type", "displayName": "Truststore Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Protocol": { "name": "SSL Protocol", "displayName": "TLS Protocol", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Keystore Type": { "name": "Keystore Type", "displayName": "Keystore Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Truststore Filename": { "name": "Truststore Filename", "displayName": "Truststore Filename", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "SINGLE", "resourceTypes": [ "FILE" ] } }, "Keystore Password": { "name": "Keystore Password", "displayName": "Keystore Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "key-password": { "name": "key-password", "displayName": "Key Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Truststore Password": { "name": "Truststore Password", "displayName": "Truststore Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Keystore Filename": { "name": "Keystore Filename", "displayName": "Keystore Filename", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "SINGLE", "resourceTypes": [ "FILE" ] } } }, "controllerServiceApis": [ { "type": "org.apache.nifi.ssl.SSLContextProvider", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } }, { "type": "org.apache.nifi.ssl.SSLContextService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } }, { "type": "org.apache.nifi.ssl.RestrictedSSLContextService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } } ], "scheduledState": "DISABLED", "bulletinLevel": "WARN", "componentType": "CONTROLLER_SERVICE", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "f6b7ad0f-da95-39c4-be8d-4ae944d8034b", "instanceIdentifier": "ca38b3a6-868b-33f0-a858-30da336cc679", "name": "JsonRecordSetWriter", "type": "org.apache.nifi.json.JsonRecordSetWriter", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-record-serialization-services-nar", "version": "2.6.0" }, "properties": { "schema-reference-reader": null, "schema-branch": null, "Allow Scientific Notation": "false", "compression-level": "1", "schema-cache": null, "Timestamp Format": null, "Date Format": null, "Pretty Print JSON": "false", "compression-format": "none", "Schema Write Strategy": "no-schema", "suppress-nulls": "never-suppress", "output-grouping": "output-array", "schema-name": "${schema.name}", "schema-registry": null, "Time Format": null, "schema-access-strategy": "inherit-record-schema", "schema-version": null, "schema-text": "${avro.schema}", "Schema Reference Writer": null }, "propertyDescriptors": { "schema-reference-reader": { "name": "schema-reference-reader", "displayName": "Schema Reference Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "schema-branch": { "name": "schema-branch", "displayName": "Schema Branch", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Allow Scientific Notation": { "name": "Allow Scientific Notation", "displayName": "Allow Scientific Notation", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "compression-level": { "name": "compression-level", "displayName": "Compression Level", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-cache": { "name": "schema-cache", "displayName": "Schema Cache", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Timestamp Format": { "name": "Timestamp Format", "displayName": "Timestamp Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Date Format": { "name": "Date Format", "displayName": "Date Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Pretty Print JSON": { "name": "Pretty Print JSON", "displayName": "Pretty Print JSON", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "compression-format": { "name": "compression-format", "displayName": "Compression Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Schema Write Strategy": { "name": "Schema Write Strategy", "displayName": "Schema Write Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "suppress-nulls": { "name": "suppress-nulls", "displayName": "Suppress Null Values", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "output-grouping": { "name": "output-grouping", "displayName": "Output Grouping", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-name": { "name": "schema-name", "displayName": "Schema Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-registry": { "name": "schema-registry", "displayName": "Schema Registry", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Time Format": { "name": "Time Format", "displayName": "Time Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-access-strategy": { "name": "schema-access-strategy", "displayName": "Schema Access Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-version": { "name": "schema-version", "displayName": "Schema Version", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-text": { "name": "schema-text", "displayName": "Schema Text", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Schema Reference Writer": { "name": "Schema Reference Writer", "displayName": "Schema Reference Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false } }, "controllerServiceApis": [ { "type": "org.apache.nifi.serialization.RecordSetWriterFactory", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } } ], "scheduledState": "DISABLED", "bulletinLevel": "WARN", "componentType": "CONTROLLER_SERVICE", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "e75c34eb-8e60-3b6b-8a29-3dd7b79776c4", "instanceIdentifier": "2d152882-2f62-3e45-8c8e-5f3127f8b931", "name": "JsonTreeReader", "type": "org.apache.nifi.json.JsonTreeReader", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-record-serialization-services-nar", "version": "2.6.0" }, "properties": { "schema-reference-reader": null, "schema-branch": null, "Max String Length": "20 MB", "schema-application-strategy": "SELECTED_PART", "Timestamp Format": null, "schema-inference-cache": null, "Date Format": null, "schema-name": "${schema.name}", "starting-field-strategy": "ROOT_NODE", "schema-registry": null, "starting-field-name": null, "Time Format": null, "schema-access-strategy": "infer-schema", "schema-version": null, "schema-text": "${avro.schema}", "Allow Comments": "false" }, "propertyDescriptors": { "schema-reference-reader": { "name": "schema-reference-reader", "displayName": "Schema Reference Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "schema-branch": { "name": "schema-branch", "displayName": "Schema Branch", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Max String Length": { "name": "Max String Length", "displayName": "Max String Length", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-application-strategy": { "name": "schema-application-strategy", "displayName": "Schema Application Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Timestamp Format": { "name": "Timestamp Format", "displayName": "Timestamp Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-inference-cache": { "name": "schema-inference-cache", "displayName": "Schema Inference Cache", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Date Format": { "name": "Date Format", "displayName": "Date Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-name": { "name": "schema-name", "displayName": "Schema Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "starting-field-strategy": { "name": "starting-field-strategy", "displayName": "Starting Field Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-registry": { "name": "schema-registry", "displayName": "Schema Registry", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "starting-field-name": { "name": "starting-field-name", "displayName": "Starting Field Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Time Format": { "name": "Time Format", "displayName": "Time Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-access-strategy": { "name": "schema-access-strategy", "displayName": "Schema Access Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-version": { "name": "schema-version", "displayName": "Schema Version", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-text": { "name": "schema-text", "displayName": "Schema Text", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Allow Comments": { "name": "Allow Comments", "displayName": "Allow Comments", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "controllerServiceApis": [ { "type": "org.apache.nifi.serialization.RecordReaderFactory", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } } ], "scheduledState": "DISABLED", "bulletinLevel": "WARN", "componentType": "CONTROLLER_SERVICE", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" }, { "identifier": "29032623-5723-3b5d-8983-07534da27fd5", "instanceIdentifier": "815ba7c0-0195-1000-0000-00002ff4760b", "name": "Kafka3ConnectionService", "comments": "", "type": "org.apache.nifi.kafka.service.Kafka3ConnectionService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-3-service-nar", "version": "2.6.0" }, "properties": { "security.protocol": "SSL", "SSL Context Service": "a43e5844-29fd-3ad4-8f84-5cfb771da897", "ack.wait.time": "5 sec", "sasl.username": null, "max.poll.records": "10000", "sasl.mechanism": "GSSAPI", "isolation.level": "read_committed", "max.block.ms": "5 sec", "bootstrap.servers": "kafka-broker-default-headless.{{ NAMESPACE }}.svc.cluster.local:9093", "kerberos-user-service": null, "sasl.kerberos.service.name": null, "default.api.timeout.ms": "60 sec" }, "propertyDescriptors": { "ack.wait.time": { "name": "ack.wait.time", "displayName": "Acknowledgment Wait Time", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "sasl.username": { "name": "sasl.username", "displayName": "SASL Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.poll.records": { "name": "max.poll.records", "displayName": "Max Poll Records", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "bootstrap.servers": { "name": "bootstrap.servers", "displayName": "Bootstrap Servers", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "sasl.kerberos.service.name": { "name": "sasl.kerberos.service.name", "displayName": "Kerberos Service Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "security.protocol": { "name": "security.protocol", "displayName": "Security Protocol", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "sasl.mechanism": { "name": "sasl.mechanism", "displayName": "SASL Mechanism", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "isolation.level": { "name": "isolation.level", "displayName": "Transaction Isolation Level", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.block.ms": { "name": "max.block.ms", "displayName": "Max Metadata Wait Time", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "kerberos-user-service": { "name": "kerberos-user-service", "displayName": "Kerberos User Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "sasl.password": { "name": "sasl.password", "displayName": "SASL Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "default.api.timeout.ms": { "name": "default.api.timeout.ms", "displayName": "Client Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "controllerServiceApis": [ { "type": "org.apache.nifi.kafka.service.api.KafkaConnectionService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-service-api-nar", "version": "2.6.0" } } ], "scheduledState": "DISABLED", "bulletinLevel": "WARN", "componentType": "CONTROLLER_SERVICE", "groupIdentifier": "5bb5c008-b432-3d6f-b99e-fedc455bd418" } ], "defaultFlowFileExpiration": "0 sec", "defaultBackPressureObjectThreshold": 10000, "defaultBackPressureDataSizeThreshold": "1 GB", "scheduledState": "ENABLED", "executionEngine": "INHERITED", "maxConcurrentTasks": 1, "statelessFlowTimeout": "1 min", "flowFileConcurrency": "UNBOUNDED", "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE", "componentType": "PROCESS_GROUP" }, "externalControllerServices": {}, "parameterContexts": {}, "flowEncodingVersion": "1.0", "parameterProviders": {}, "latest": false } diff --git a/demos/nifi-kafka-druid-water-level-data/serviceaccount.yaml b/demos/nifi-kafka-druid-water-level-data/serviceaccount.yaml index 8a2d0b19..d7c0b939 100644 --- a/demos/nifi-kafka-druid-water-level-data/serviceaccount.yaml +++ b/demos/nifi-kafka-druid-water-level-data/serviceaccount.yaml @@ -3,7 +3,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: demo-serviceaccount - namespace: default --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -12,7 +11,7 @@ metadata: subjects: - kind: ServiceAccount name: demo-serviceaccount - namespace: default + namespace: {{ NAMESPACE }} roleRef: kind: ClusterRole name: demo-clusterrole diff --git a/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc b/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc index 9b2053d9..7db78cbd 100644 --- a/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc +++ b/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc @@ -111,7 +111,7 @@ You can execute a command on the Kafka broker to list the available topics as fo [source,console] ---- -$ kubectl k exec kafka-broker-default-0 -c kafka -- \ +$ kubectl exec kafka-broker-default-0 -c kafka -- \ /stackable/kafka/bin/kafka-topics.sh \ --describe \ --bootstrap-server kafka-broker-default-headless.default.svc.cluster.local:9093 \ diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index 69aefb65..845c735e 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -297,6 +297,9 @@ stacks: memory: 42330Mi pvc: 75Gi parameters: + - name: stackLabel + description: Stack label to add to various components + default: nifi-kafka-druid-superset-s3 - name: nifiAdminPassword description: Password of the NiFI admin user default: adminadmin From f36fa9b1199af853ce6a6ca5664d5cf2513ae106 Mon Sep 17 00:00:00 2001 From: xeniape Date: Mon, 9 Mar 2026 12:31:30 +0100 Subject: [PATCH 05/22] spark-k8s-anomaly-detection-taxi-data demo in non-default ns and with demo labels --- .../spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml | 2 +- stacks/_templates/postgresql-hive.yaml | 1 + stacks/spark-trino-superset-s3/s3-connection.yaml | 2 +- stacks/stacks-v2.yaml | 3 +++ 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml b/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml index 9b11373c..fe0bade1 100644 --- a/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml +++ b/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml @@ -17,7 +17,7 @@ spec: cd /tmp for month in 2020-09 2020-10 2020-11 2020-12; do curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/fhvhv_tripdata_$month.parquet - mc alias set minio https://minio.default.svc.cluster.local:9000/ $(cat /minio-s3-credentials/accessKey) $(cat /minio-s3-credentials/secretKey) + mc alias set minio https://minio.{{ NAMESPACE }}.svc.cluster.local:9000/ $(cat /minio-s3-credentials/accessKey) $(cat /minio-s3-credentials/secretKey) mc cp fhvhv_tripdata_$month.parquet minio/demo/ny-taxi-data/raw/ mc mb --ignore-existing minio/prediction; done diff --git a/stacks/_templates/postgresql-hive.yaml b/stacks/_templates/postgresql-hive.yaml index 19e82517..12693c37 100644 --- a/stacks/_templates/postgresql-hive.yaml +++ b/stacks/_templates/postgresql-hive.yaml @@ -8,6 +8,7 @@ version: 18.1.3 # 18.0.0 options: commonLabels: stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} auth: username: hive password: hive diff --git a/stacks/spark-trino-superset-s3/s3-connection.yaml b/stacks/spark-trino-superset-s3/s3-connection.yaml index 980a3626..e52f7183 100644 --- a/stacks/spark-trino-superset-s3/s3-connection.yaml +++ b/stacks/spark-trino-superset-s3/s3-connection.yaml @@ -5,7 +5,7 @@ metadata: name: minio spec: # Hostname must match secret op's certificate SAN - host: minio.default.svc.cluster.local + host: minio.{{ NAMESPACE }}.svc.cluster.local port: 9000 accessStyle: Path credentials: diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index 845c735e..2b9fed90 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -345,6 +345,9 @@ stacks: memory: 15910Mi pvc: 36Gi parameters: + - name: stackLabel + description: Stack label to add to various components + default: spark-trino-superset-s3 - name: trinoAdminPassword description: Password of the Trino admin user default: adminadmin From 35db02e96d50240b860d08211fdcc693e724c266 Mon Sep 17 00:00:00 2001 From: xeniape Date: Mon, 9 Mar 2026 13:26:59 +0100 Subject: [PATCH 06/22] trino-iceberg demo in non-default ns and with demo labels --- .../rendered-chart.yaml | 92 +++++++++---------- stacks/stacks-v2.yaml | 5 +- stacks/trino-iceberg/s3-connection.yaml | 2 +- 3 files changed, 51 insertions(+), 48 deletions(-) diff --git a/stacks/_templates/minio-distributed-small-tls/rendered-chart.yaml b/stacks/_templates/minio-distributed-small-tls/rendered-chart.yaml index 728b1871..35d2fe7c 100644 --- a/stacks/_templates/minio-distributed-small-tls/rendered-chart.yaml +++ b/stacks/_templates/minio-distributed-small-tls/rendered-chart.yaml @@ -36,7 +36,7 @@ data: set -e # Have script exit in the event of a failed command. MC_CONFIG_DIR="/etc/minio/mc/" MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - + # connectToMinio # Use a check-sleep-check loop to wait for MinIO service to be available connectToMinio() { @@ -64,7 +64,7 @@ data: set -e # reset `e` as active return 0 } - + # checkBucketExists ($bucket) # Check if the bucket exists, by using the exit code of `mc ls` checkBucketExists() { @@ -72,7 +72,7 @@ data: CMD=$(${MC} stat myminio/$BUCKET >/dev/null 2>&1) return $? } - + # createBucket ($bucket, $policy, $purge) # Ensure bucket exists, purging if asked to createBucket() { @@ -81,7 +81,7 @@ data: PURGE=$3 VERSIONING=$4 OBJECTLOCKING=$5 - + # Purge the bucket, if set & exists # Since PURGE is user input, check explicitly for `true` if [ $PURGE = true ]; then @@ -94,7 +94,7 @@ data: echo "Bucket '$BUCKET' does not exist, skipping purge." fi fi - + # Create the bucket if it does not exist and set objectlocking if enabled (NOTE: versioning will be not changed if OBJECTLOCKING is set because it enables versioning to the Buckets created) if ! checkBucketExists $BUCKET; then if [ ! -z $OBJECTLOCKING ]; then @@ -112,7 +112,7 @@ data: echo "Bucket '$BUCKET' already exists." fi fi - + # set versioning for bucket if objectlocking is disabled or not set if [ $OBJECTLOCKING = false ]; then if [ ! -z $VERSIONING ]; then @@ -127,32 +127,32 @@ data: else echo "Bucket '$BUCKET' versioning unchanged." fi - + # At this point, the bucket should exist, skip checking for existence # Set policy on the bucket echo "Setting policy of bucket '$BUCKET' to '$POLICY'." ${MC} anonymous set $POLICY myminio/$BUCKET } - + # Try connecting to MinIO instance scheme=https connectToMinio $scheme - - - + + + # Create the buckets createBucket lakehouse "public" false false false - + add-user: |- #!/bin/sh set -e ; # Have script exit in the event of a failed command. MC_CONFIG_DIR="/etc/minio/mc/" MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - + # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. # Special characters for example : ',",<,>,{,} MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_tmp" - + # connectToMinio # Use a check-sleep-check loop to wait for MinIO service to be available connectToMinio() { @@ -179,14 +179,14 @@ data: set -e ; # reset `e` as active return 0 } - + # checkUserExists () # Check if the user exists, by using the exit code of `mc admin user info` checkUserExists() { CMD=$(${MC} admin user info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) return $? } - + # createUser ($policy) createUser() { POLICY=$1 @@ -210,7 +210,7 @@ data: fi #clean up credentials files. rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - + # set policy for user if [ ! -z $POLICY -a $POLICY != " " ] ; then echo "Adding policy '$POLICY' for '$USER'" @@ -221,24 +221,24 @@ data: echo "User '$USER' has no policy attached." fi } - + # Try connecting to MinIO instance scheme=https connectToMinio $scheme - - - + + + # Create the users echo console > $MINIO_ACCESSKEY_SECRETKEY_TMP echo console123 >> $MINIO_ACCESSKEY_SECRETKEY_TMP createUser consoleAdmin - + add-policy: |- #!/bin/sh set -e ; # Have script exit in the event of a failed command. MC_CONFIG_DIR="/etc/minio/mc/" MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - + # connectToMinio # Use a check-sleep-check loop to wait for MinIO service to be available connectToMinio() { @@ -265,7 +265,7 @@ data: set -e ; # reset `e` as active return 0 } - + # checkPolicyExists ($policy) # Check if the policy exists, by using the exit code of `mc admin policy info` checkPolicyExists() { @@ -273,12 +273,12 @@ data: CMD=$(${MC} admin policy info myminio $POLICY > /dev/null 2>&1) return $? } - + # createPolicy($name, $filename) createPolicy () { NAME=$1 FILENAME=$2 - + # Create the name if it does not exist echo "Checking policy: $NAME (in /config/$FILENAME.json)" if ! checkPolicyExists $NAME ; then @@ -287,25 +287,25 @@ data: echo "Policy '$NAME' already exists." fi ${MC} admin policy create myminio $NAME /config/$FILENAME.json - + } - + # Try connecting to MinIO instance scheme=https connectToMinio $scheme - - - + + + add-svcacct: |- #!/bin/sh set -e ; # Have script exit in the event of a failed command. MC_CONFIG_DIR="/etc/minio/mc/" MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - + # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. # Special characters for example : ',",<,>,{,} MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_svcacct_tmp" - + # connectToMinio # Use a check-sleep-check loop to wait for MinIO service to be available connectToMinio() { @@ -332,14 +332,14 @@ data: set -e ; # reset `e` as active return 0 } - + # checkSvcacctExists () # Check if the svcacct exists, by using the exit code of `mc admin user svcacct info` checkSvcacctExists() { CMD=$(${MC} admin user svcacct info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) return $? } - + # createSvcacct ($user) createSvcacct () { USER=$1 @@ -370,19 +370,19 @@ data: #clean up credentials files. rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP } - + # Try connecting to MinIO instance scheme=https connectToMinio $scheme - - - + + + custom-command: |- #!/bin/sh set -e ; # Have script exit in the event of a failed command. MC_CONFIG_DIR="/etc/minio/mc/" MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - + # connectToMinio # Use a check-sleep-check loop to wait for MinIO service to be available connectToMinio() { @@ -409,14 +409,14 @@ data: set -e ; # reset `e` as active return 0 } - + # runCommand ($@) # Run custom mc command runCommand() { ${MC} "$@" return $? } - + # Try connecting to MinIO instance scheme=https connectToMinio $scheme @@ -543,7 +543,7 @@ spec: mkdir -p /etc/minio/certs/CAs cp -v /etc/minio/original_certs/ca.crt /etc/minio/certs/CAs/public.crt - /usr/bin/docker-entrypoint.sh minio server https://minio-{0...1}.minio-svc.default.svc.cluster.local/export -S /etc/minio/certs/ --address :9000 --console-address :9001 + /usr/bin/docker-entrypoint.sh minio server https://minio-{0...1}.minio-svc.{{ NAMESPACE }}.svc.cluster.local/export -S /etc/minio/certs/ --address :9000 --console-address :9001 volumeMounts: - name: export mountPath: /export @@ -573,8 +573,8 @@ spec: requests: cpu: 500m memory: 1Gi - securityContext: - readOnlyRootFilesystem: false + securityContext: + readOnlyRootFilesystem: false volumes: - name: minio-user secret: @@ -629,7 +629,7 @@ spec: release: minio stackable.tech/vendor: Stackable spec: - restartPolicy: OnFailure + restartPolicy: OnFailure volumes: - name: etc-path emptyDir: {} diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index 2b9fed90..1230706a 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -424,12 +424,15 @@ stacks: - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/trino-iceberg/s3-connection.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/trino-iceberg/hive-metastores.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/trino-iceberg/trino.yaml - supportedNamespaces: ["default"] # until namespace can be templates, the minio FQDN hard-codes the namespace + supportedNamespaces: [] resourceRequests: cpu: 6000m # Measured 5600m memory: 21Gi pvc: 110Gi # 100Gi for MinIO parameters: + - name: stackLabel + description: Stack label to add to various components + default: trino-iceberg - name: trinoAdminPassword description: Password of the Trino admin user default: adminadmin diff --git a/stacks/trino-iceberg/s3-connection.yaml b/stacks/trino-iceberg/s3-connection.yaml index 411d7a4d..610e6541 100644 --- a/stacks/trino-iceberg/s3-connection.yaml +++ b/stacks/trino-iceberg/s3-connection.yaml @@ -4,7 +4,7 @@ kind: S3Connection metadata: name: minio spec: - host: minio.default.svc.cluster.local + host: minio.{{ NAMESPACE }}.svc.cluster.local port: 9000 accessStyle: Path credentials: From 13836ab48c0f4de720cafe59de9fef4d8f35c20d Mon Sep 17 00:00:00 2001 From: xeniape Date: Mon, 9 Mar 2026 14:07:50 +0100 Subject: [PATCH 07/22] trino-taxi-data demo in non-default ns and with demo labels --- demos/trino-taxi-data/load-test-data.yaml | 2 +- stacks/stacks-v2.yaml | 5 ++++- stacks/trino-superset-s3/s3-connection.yaml | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/demos/trino-taxi-data/load-test-data.yaml b/demos/trino-taxi-data/load-test-data.yaml index 63892d4f..1eb1c8f8 100644 --- a/demos/trino-taxi-data/load-test-data.yaml +++ b/demos/trino-taxi-data/load-test-data.yaml @@ -17,7 +17,7 @@ spec: # Copy the CA cert from the "tls" SecretClass cp -v /etc/minio/mc/original_certs/ca.crt /.mc/certs/CAs/public.crt - MINIO_ENDPOINT="https://minio.default.svc.cluster.local:9000/" + MINIO_ENDPOINT="https://minio.{{ NAMESPACE }}.svc.cluster.local:9000/" MINIO_ACCESS_KEY=$(cat /minio-s3-credentials/accessKey) MINIO_SECRET_KEY=$(cat /minio-s3-credentials/secretKey) diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index 1230706a..dcafa0ce 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -384,12 +384,15 @@ stacks: - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/trino-superset-s3/hive-metastore.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/trino-superset-s3/trino.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/trino-superset-s3/superset.yaml - supportedNamespaces: [default] # until namespace can be templates, the minio FQDN hard-codes the namespace + supportedNamespaces: [] resourceRequests: cpu: 6800m memory: 15822Mi pvc: 28Gi parameters: + - name: stackLabel + description: Stack label to add to various components + default: trino-superset-s3 - name: trinoAdminPassword description: Password of the Trino admin user default: adminadmin diff --git a/stacks/trino-superset-s3/s3-connection.yaml b/stacks/trino-superset-s3/s3-connection.yaml index 411d7a4d..610e6541 100644 --- a/stacks/trino-superset-s3/s3-connection.yaml +++ b/stacks/trino-superset-s3/s3-connection.yaml @@ -4,7 +4,7 @@ kind: S3Connection metadata: name: minio spec: - host: minio.default.svc.cluster.local + host: minio.{{ NAMESPACE }}.svc.cluster.local port: 9000 accessStyle: Path credentials: From 065b54006c1319c6a3ef03d98e5b01e977893c7d Mon Sep 17 00:00:00 2001 From: xeniape Date: Mon, 9 Mar 2026 17:18:02 +0100 Subject: [PATCH 08/22] logging demo in non-default ns and with demo labels --- stacks/_templates/opensearch-dashboards.yaml | 2 ++ stacks/_templates/vector-aggregator.yaml | 2 ++ stacks/stacks-v2.yaml | 3 +++ 3 files changed, 7 insertions(+) diff --git a/stacks/_templates/opensearch-dashboards.yaml b/stacks/_templates/opensearch-dashboards.yaml index fe0b511a..5ffcb29e 100644 --- a/stacks/_templates/opensearch-dashboards.yaml +++ b/stacks/_templates/opensearch-dashboards.yaml @@ -11,6 +11,7 @@ options: tag: "{{ opensearchVersion }}-stackable{{ stackableReleaseVersion }}" labels: stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} service: type: NodePort port: 5601 @@ -19,6 +20,7 @@ options: stackable.tech/logging-credentials-secret: opensearch-user labels: stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} opensearchHosts: null # Use the discovery ConfigMap instead extraEnvs: - name: OPENSEARCH_HOSTS diff --git a/stacks/_templates/vector-aggregator.yaml b/stacks/_templates/vector-aggregator.yaml index 469ec23f..80593204 100644 --- a/stacks/_templates/vector-aggregator.yaml +++ b/stacks/_templates/vector-aggregator.yaml @@ -8,8 +8,10 @@ version: 0.49.0 # app version 0.52.0 options: commonLabels: stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} podLabels: stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} role: Aggregator customConfig: sources: diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index dcafa0ce..e6ed2cd6 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -93,6 +93,9 @@ stacks: memory: 3147Mi pvc: 24Gi parameters: + - name: stackLabel + description: Stack label to add to various components + default: logging - name: openSearchAdminPassword description: Password of the OpenSearch admin user default: adminadmin From ec9f7465c6308f8535332ec9d64b29ed32b4cacb Mon Sep 17 00:00:00 2001 From: xeniape Date: Mon, 9 Mar 2026 21:32:21 +0100 Subject: [PATCH 09/22] signal-processing demo in non-default ns and with demo labels --- demos/signal-processing/DownloadAndWriteToDB.json | 2 +- .../signal-processing/create-nifi-ingestion-job.yaml | 11 ++++++----- demos/signal-processing/create-timescale-tables.yaml | 8 ++++---- demos/signal-processing/serviceaccount.yaml | 3 +-- stacks/_templates/postgresql-timescaledb.yaml | 1 + stacks/signal-processing/grafana.yaml | 3 ++- stacks/signal-processing/jupyterhub.yaml | 3 +++ stacks/signal-processing/serviceaccount.yaml | 1 + stacks/signal-processing/tsdb.ipynb | 2 +- stacks/stacks-v2.yaml | 3 +++ 10 files changed, 23 insertions(+), 14 deletions(-) diff --git a/demos/signal-processing/DownloadAndWriteToDB.json b/demos/signal-processing/DownloadAndWriteToDB.json index 66fc8599..42b38d8e 100644 --- a/demos/signal-processing/DownloadAndWriteToDB.json +++ b/demos/signal-processing/DownloadAndWriteToDB.json @@ -741,7 +741,7 @@ "kerberos-credentials-service": null, "dbcp-max-conn-lifetime": "-1", "Validation-query": "select count(*) from conditions;", - "Database Connection URL": "jdbc:postgresql://postgresql-timescaledb.default.svc.cluster.local:5432/tsdb", + "Database Connection URL": "jdbc:postgresql://postgresql-timescaledb.{{ NAMESPACE }}.svc.cluster.local:5432/tsdb", "dbcp-time-between-eviction-runs": "-1", "Database User": "admin", "kerberos-user-service": null, diff --git a/demos/signal-processing/create-nifi-ingestion-job.yaml b/demos/signal-processing/create-nifi-ingestion-job.yaml index 146c8840..dd67cedf 100644 --- a/demos/signal-processing/create-nifi-ingestion-job.yaml +++ b/demos/signal-processing/create-nifi-ingestion-job.yaml @@ -17,15 +17,15 @@ spec: - -c - | echo 'Waiting for timescaleDB job to be created' - until kubectl get job create-timescale-tables-job 2>/dev/null; do sleep 5; done - kubectl wait --for=create job/create-timescale-tables-job --timeout=30m + until kubectl get job create-timescale-tables-job -n {{ NAMESPACE }} 2>/dev/null; do sleep 5; done + kubectl wait --for=create job/create-timescale-tables-job --timeout=30m -n {{ NAMESPACE }} echo 'Waiting for timescaleDB job to be completed' - kubectl wait --for=condition=complete job/create-timescale-tables-job --timeout=30m + kubectl wait --for=condition=complete job/create-timescale-tables-job --timeout=30m -n {{ NAMESPACE }} echo 'Waiting for NiFi to be created' - kubectl wait --for=create pod/nifi-node-default-0 --timeout=30m + kubectl wait --for=create pod/nifi-node-default-0 --timeout=30m -n {{ NAMESPACE }} echo 'Waiting for NiFi to be ready' - kubectl wait --for=condition=Ready pod/nifi-node-default-0 --timeout=30m + kubectl wait --for=condition=Ready pod/nifi-node-default-0 --timeout=30m -n {{ NAMESPACE }} containers: - name: create-nifi-ingestion-job image: oci.stackable.tech/sdp/testing-tools/nifi:0.3.0-stackable0.0.0-dev @@ -87,6 +87,7 @@ data: service_login(username=USERNAME, password=PASSWORD) print("Logged in") + # This file path needs adjustment for versioned demos on release branch response = requests.get("https://raw.githubusercontent.com/stackabletech/demos/refs/heads/main/demos/signal-processing/DownloadAndWriteToDB.json") filename = "/tmp/DownloadAndWriteToDB.json" diff --git a/demos/signal-processing/create-timescale-tables.yaml b/demos/signal-processing/create-timescale-tables.yaml index efa8c7d6..9e9ae27a 100644 --- a/demos/signal-processing/create-timescale-tables.yaml +++ b/demos/signal-processing/create-timescale-tables.yaml @@ -17,10 +17,10 @@ spec: - -c - | echo 'Waiting for timescaleDB to be created' - kubectl wait --for=create pod/postgresql-timescaledb-0 --timeout=30m + kubectl wait --for=create pod/postgresql-timescaledb-0 --timeout=30m -n {{ NAMESPACE }} echo 'Waiting for timescaleDB to be ready' - kubectl wait --for=condition=ready pod/postgresql-timescaledb-0 --timeout=30m + kubectl wait --for=condition=ready pod/postgresql-timescaledb-0 --timeout=30m -n {{ NAMESPACE }} containers: - name: create-timescale-tables-job image: postgres @@ -32,14 +32,14 @@ spec: - | export PGPASSWORD=$(cat /timescale-admin-credentials/password) echo 'Submitting DDL...' - psql -U admin -h postgresql-timescaledb.default.svc.cluster.local postgres \ + psql -U admin -h postgresql-timescaledb.{{ NAMESPACE }}.svc.cluster.local postgres \ -c '\x' -c 'CREATE DATABASE tsdb' \ -c '\c tsdb' \ -f /tmp/sql/timescaledb.sql \ -c 'select count(*) from conditions' \ -c '\q' echo 'Creating extension as superuser...' - psql -U postgres -h postgresql-timescaledb.default.svc.cluster.local postgres \ + psql -U postgres -h postgresql-timescaledb.{{ NAMESPACE }}.svc.cluster.local postgres \ -c '\x' \ -c '\c tsdb' \ -c 'CREATE EXTENSION timescaledb_toolkit' \ diff --git a/demos/signal-processing/serviceaccount.yaml b/demos/signal-processing/serviceaccount.yaml index 8a2d0b19..d7c0b939 100644 --- a/demos/signal-processing/serviceaccount.yaml +++ b/demos/signal-processing/serviceaccount.yaml @@ -3,7 +3,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: demo-serviceaccount - namespace: default --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -12,7 +11,7 @@ metadata: subjects: - kind: ServiceAccount name: demo-serviceaccount - namespace: default + namespace: {{ NAMESPACE }} roleRef: kind: ClusterRole name: demo-clusterrole diff --git a/stacks/_templates/postgresql-timescaledb.yaml b/stacks/_templates/postgresql-timescaledb.yaml index e1f3f2dc..61611a90 100644 --- a/stacks/_templates/postgresql-timescaledb.yaml +++ b/stacks/_templates/postgresql-timescaledb.yaml @@ -8,6 +8,7 @@ version: 0.33.1 options: podLabels: stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} replicaCount: 1 secrets: credentials: diff --git a/stacks/signal-processing/grafana.yaml b/stacks/signal-processing/grafana.yaml index 65c7b3f1..85b4a6d2 100644 --- a/stacks/signal-processing/grafana.yaml +++ b/stacks/signal-processing/grafana.yaml @@ -8,6 +8,7 @@ version: 10.1.2 # 12.2.0 options: extraLabels: stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} grafana.ini: analytics: check_for_updates: false @@ -38,7 +39,7 @@ options: datasources: - name: postgres type: postgres - url: postgresql-timescaledb.default.svc.cluster.local:5432 + url: postgresql-timescaledb.{{ NAMESPACE }}.svc.cluster.local:5432 user: admin secureJsonData: password: {{ postgresAdminPassword }} diff --git a/stacks/signal-processing/jupyterhub.yaml b/stacks/signal-processing/jupyterhub.yaml index a88fa324..dfced547 100644 --- a/stacks/signal-processing/jupyterhub.yaml +++ b/stacks/signal-processing/jupyterhub.yaml @@ -17,6 +17,7 @@ options: authenticator_class: dummy labels: stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} proxy: service: type: NodePort @@ -39,9 +40,11 @@ options: enabled: false extraLabels: stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} initContainers: - name: download-notebook image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev + # This file path needs adjustment for versioned demos on release branch command: ['sh', '-c', 'curl https://raw.githubusercontent.com/stackabletech/demos/main/stacks/signal-processing/tsdb.ipynb -o /notebook/tsdb.ipynb'] volumeMounts: - mountPath: /notebook diff --git a/stacks/signal-processing/serviceaccount.yaml b/stacks/signal-processing/serviceaccount.yaml index f8298b4b..d1e4901e 100644 --- a/stacks/signal-processing/serviceaccount.yaml +++ b/stacks/signal-processing/serviceaccount.yaml @@ -11,6 +11,7 @@ metadata: subjects: - kind: ServiceAccount name: spark + namespace: {{ NAMESPACE }} roleRef: kind: Role name: spark-role diff --git a/stacks/signal-processing/tsdb.ipynb b/stacks/signal-processing/tsdb.ipynb index 17e87513..d5581578 100644 --- a/stacks/signal-processing/tsdb.ipynb +++ b/stacks/signal-processing/tsdb.ipynb @@ -68,7 +68,7 @@ "metadata": {}, "outputs": [], "source": [ - "engine = sqlalchemy.create_engine(\"postgresql://admin:adminadmin@postgresql-timescaledb.default.svc.cluster.local:5432/tsdb\")\n", + "engine = sqlalchemy.create_engine(\"postgresql://admin:adminadmin@postgresql-timescaledb.{{ NAMESPACE }}.svc.cluster.local:5432/tsdb\")\n", "last_time = \"1970-01-01\"\n", "cols = ['r1', 'r2', 'r3', 'r4', 'r5', 'r6', 'r7', 'r8', 'r9', 'r10', 'r11', 'r12', 'r13', 'r14']" ] diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index e6ed2cd6..48cf20cf 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -666,6 +666,9 @@ stacks: - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/signal-processing/spark_driver_service.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/signal-processing/jupyterhub.yaml parameters: + - name: stackLabel + description: Stack label to add to various components + default: signal-processing - name: nifiAdminPassword description: Password of the NiFI admin user default: adminadmin From fe87f3993e80b871761b0eeec993d5fd4fe5410c Mon Sep 17 00:00:00 2001 From: xeniape Date: Tue, 10 Mar 2026 08:49:00 +0100 Subject: [PATCH 10/22] jupyterhub-keycloak demo in non-default ns and with demo labels --- stacks/_templates/minio.yaml | 2 ++ stacks/jupyterhub-keycloak/jupyterhub-native-auth.yaml | 2 ++ stacks/jupyterhub-keycloak/jupyterhub.yaml | 2 ++ stacks/jupyterhub-keycloak/keycloak-serviceaccount.yaml | 3 +-- stacks/jupyterhub-keycloak/keycloak.yaml | 7 ++++++- stacks/jupyterhub-keycloak/serviceaccount.yaml | 1 + stacks/stacks-v2.yaml | 3 +++ 7 files changed, 17 insertions(+), 3 deletions(-) diff --git a/stacks/_templates/minio.yaml b/stacks/_templates/minio.yaml index 5d475afc..18147a4c 100644 --- a/stacks/_templates/minio.yaml +++ b/stacks/_templates/minio.yaml @@ -8,8 +8,10 @@ version: 5.4.0 # RELEASE.2024-12-18T13-15-44Z options: additionalLabels: stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} podLabels: stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} rootUser: admin rootPassword: {{ minioAdminPassword }} mode: standalone diff --git a/stacks/jupyterhub-keycloak/jupyterhub-native-auth.yaml b/stacks/jupyterhub-keycloak/jupyterhub-native-auth.yaml index dc04171a..41b44527 100644 --- a/stacks/jupyterhub-keycloak/jupyterhub-native-auth.yaml +++ b/stacks/jupyterhub-keycloak/jupyterhub-native-auth.yaml @@ -1,3 +1,4 @@ +# TODO: This file does not seem to be used, remove it? --- releaseName: jupyterhub name: jupyterhub @@ -39,6 +40,7 @@ options: enabled: false extraLabels: stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} profileList: - display_name: "Default" description: "Default profile" diff --git a/stacks/jupyterhub-keycloak/jupyterhub.yaml b/stacks/jupyterhub-keycloak/jupyterhub.yaml index 0b365e02..2e6a0e6a 100644 --- a/stacks/jupyterhub-keycloak/jupyterhub.yaml +++ b/stacks/jupyterhub-keycloak/jupyterhub.yaml @@ -149,9 +149,11 @@ options: enabled: false extraLabels: stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} initContainers: - name: download-notebook image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev + # This file path needs adjustment for versioned demos on release branch command: ['sh', '-c', 'curl https://raw.githubusercontent.com/stackabletech/demos/main/stacks/jupyterhub-keycloak/process-s3.ipynb -o /notebook/process-s3.ipynb'] volumeMounts: - mountPath: /notebook diff --git a/stacks/jupyterhub-keycloak/keycloak-serviceaccount.yaml b/stacks/jupyterhub-keycloak/keycloak-serviceaccount.yaml index 43a5837a..2789a530 100644 --- a/stacks/jupyterhub-keycloak/keycloak-serviceaccount.yaml +++ b/stacks/jupyterhub-keycloak/keycloak-serviceaccount.yaml @@ -3,7 +3,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: keycloak-serviceaccount - namespace: default --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -12,7 +11,7 @@ metadata: subjects: - kind: ServiceAccount name: keycloak-serviceaccount - namespace: default + namespace: {{ NAMESPACE }} roleRef: kind: ClusterRole name: keycloak-clusterrole diff --git a/stacks/jupyterhub-keycloak/keycloak.yaml b/stacks/jupyterhub-keycloak/keycloak.yaml index 120a9b9b..0069d741 100644 --- a/stacks/jupyterhub-keycloak/keycloak.yaml +++ b/stacks/jupyterhub-keycloak/keycloak.yaml @@ -61,18 +61,23 @@ spec: while : do echo "Determining Keycloak public reachable address" - KEYCLOAK_ADDRESS=$(kubectl get svc keycloak -o json | jq -r --argfile endpoints <(kubectl get endpoints keycloak -o json) --argfile nodes <(kubectl get nodes -o json) '($nodes.items[] | select(.metadata.name == $endpoints.subsets[].addresses[].nodeName) | .status.addresses | map(select(.type == "ExternalIP" or .type == "InternalIP")) | min_by(.type) | .address | tostring) + ":" + (.spec.ports[] | select(.name == "https") | .nodePort | tostring)') + KEYCLOAK_ADDRESS=$(kubectl get svc keycloak -n {{ NAMESPACE }} -o json | jq -r --slurpfile endpoints <(kubectl get endpoints keycloak -n {{ NAMESPACE }} -o json) --slurpfile nodes <(kubectl get nodes -o json) '($nodes[0].items[] | select(.metadata.name == $endpoints[0].subsets[].addresses[].nodeName) | .status.addresses | map(select(.type == "ExternalIP" or .type == "InternalIP")) | min_by(.type) | .address | tostring) + ":" + (.spec.ports[] | select(.name == "https") | .nodePort | tostring)') echo "Found Keycloak running at $KEYCLOAK_ADDRESS" if [ ! -z "$KEYCLOAK_ADDRESS" ]; then KEYCLOAK_HOSTNAME="$(echo $KEYCLOAK_ADDRESS | grep -oP '^[^:]+')" KEYCLOAK_PORT="$(echo $KEYCLOAK_ADDRESS | grep -oP '[0-9]+$')" + STACK_LABEL=$(kubectl get svc keycloak -n {{ NAMESPACE }} -o json | jq '.metadata.labels."stackable.tech/stack"') + cat << EOF | kubectl apply -f - apiVersion: v1 kind: ConfigMap metadata: name: keycloak-address + labels: + stackable.tech/vendor: Stackable + stackable.tech/stack: $STACK_LABEL data: keycloakAddress: "$KEYCLOAK_HOSTNAME:$KEYCLOAK_PORT" keycloakNodeIp: "$KEYCLOAK_HOSTNAME" diff --git a/stacks/jupyterhub-keycloak/serviceaccount.yaml b/stacks/jupyterhub-keycloak/serviceaccount.yaml index f8298b4b..d1e4901e 100644 --- a/stacks/jupyterhub-keycloak/serviceaccount.yaml +++ b/stacks/jupyterhub-keycloak/serviceaccount.yaml @@ -11,6 +11,7 @@ metadata: subjects: - kind: ServiceAccount name: spark + namespace: {{ NAMESPACE }} roleRef: kind: Role name: spark-role diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index 48cf20cf..14a23af3 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -707,6 +707,9 @@ stacks: - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/jupyterhub-keycloak/serviceaccount.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/jupyterhub-keycloak/s3-connection.yaml parameters: + - name: stackLabel + description: Stack label to add to various components + default: jupyterhub-keycloak - name: keycloakAdminPassword description: Password of the Keycloak admin user default: adminadmin From 13347419a818e1fb5df326bec4afe2cad45bf500 Mon Sep 17 00:00:00 2001 From: xeniape Date: Tue, 10 Mar 2026 10:25:26 +0100 Subject: [PATCH 11/22] opensearch demo in non-default ns and with demo labels --- stacks/opensearch-rag/jupyterlab.yaml | 2 ++ stacks/opensearch-rag/ollama.yaml | 3 --- stacks/stacks-v2.yaml | 3 +++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/stacks/opensearch-rag/jupyterlab.yaml b/stacks/opensearch-rag/jupyterlab.yaml index 9eabc005..559c60a7 100644 --- a/stacks/opensearch-rag/jupyterlab.yaml +++ b/stacks/opensearch-rag/jupyterlab.yaml @@ -12,11 +12,13 @@ spec: matchLabels: app: jupyterlab stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} template: metadata: labels: app: jupyterlab stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} spec: serviceAccountName: default containers: diff --git a/stacks/opensearch-rag/ollama.yaml b/stacks/opensearch-rag/ollama.yaml index 67aa4674..1ccf410b 100644 --- a/stacks/opensearch-rag/ollama.yaml +++ b/stacks/opensearch-rag/ollama.yaml @@ -4,7 +4,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: ollama - namespace: default labels: app.kubernetes.io/name: ollama app.kubernetes.io/instance: ollama @@ -15,7 +14,6 @@ apiVersion: v1 kind: Service metadata: name: ollama - namespace: default labels: app.kubernetes.io/name: ollama app.kubernetes.io/instance: ollama @@ -35,7 +33,6 @@ apiVersion: apps/v1 kind: Deployment metadata: name: ollama - namespace: default labels: app.kubernetes.io/name: ollama app.kubernetes.io/instance: ollama diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index 14a23af3..778168f0 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -744,6 +744,9 @@ stacks: memory: 16Gi pvc: 10Gi parameters: + - name: stackLabel + description: Stack label to add to various components + default: opensearch-rag - name: openSearchAdminPassword description: Password of the OpenSearch admin user default: adminadmin From 8c0e4c469980a64df25166a29d42b68a23baf33f Mon Sep 17 00:00:00 2001 From: xeniape Date: Tue, 10 Mar 2026 14:44:30 +0100 Subject: [PATCH 12/22] monitoring in non-default ns and with demo labels --- stacks/monitoring/prometheus.yaml | 6 +++++- stacks/stacks-v2.yaml | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/stacks/monitoring/prometheus.yaml b/stacks/monitoring/prometheus.yaml index 8fcd876d..23f7577b 100644 --- a/stacks/monitoring/prometheus.yaml +++ b/stacks/monitoring/prometheus.yaml @@ -6,8 +6,12 @@ repo: url: https://prometheus-community.github.io/helm-charts version: 77.12.0 # 0.85.0 options: - commonMetaLabels: + commonLabels: stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} + prometheusOperator: + kubeletService: + namespace: {{ NAMESPACE }} prometheus: prometheusSpec: storageSpec: diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index 778168f0..b75ccc44 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -64,6 +64,9 @@ stacks: memory: 1714Mi pvc: 110Gi parameters: + - name: stackLabel + description: Stack label to add to various components + default: monitoring - name: grafanaAdminPassword description: Password of the Grafana admin user default: adminadmin From 116668baff3477f91ca51eb5149e74a13988a527 Mon Sep 17 00:00:00 2001 From: xeniape Date: Tue, 10 Mar 2026 16:14:30 +0100 Subject: [PATCH 13/22] observability in non-default ns and with demo labels --- stacks/observability/grafana-tempo.yaml | 4 ++++ stacks/observability/grafana.yaml | 5 ++++- stacks/observability/jaeger.yaml | 5 +++-- stacks/observability/opentelemetry-operator.yaml | 3 +++ stacks/stacks-v2.yaml | 3 +++ 5 files changed, 17 insertions(+), 3 deletions(-) diff --git a/stacks/observability/grafana-tempo.yaml b/stacks/observability/grafana-tempo.yaml index 4f5e27da..d6f599be 100644 --- a/stacks/observability/grafana-tempo.yaml +++ b/stacks/observability/grafana-tempo.yaml @@ -12,6 +12,10 @@ repo: # helm search repo grafana/tempo version: 1.24.0 # 2.9.0 options: + global: + commonLabels: + stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} tempo: receivers: otlp: diff --git a/stacks/observability/grafana.yaml b/stacks/observability/grafana.yaml index 58a3f180..8319b97c 100644 --- a/stacks/observability/grafana.yaml +++ b/stacks/observability/grafana.yaml @@ -11,6 +11,9 @@ repo: # helm search repo grafana/grafana version: 10.1.4 # 12.2.1 options: + extraLabels: + stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} admin: existingSecret: grafana-admin-credentials service: @@ -66,7 +69,7 @@ options: # Mount the certificate generated by the secret-operator - name: tls mountPath: /etc/grafana/certs/ - extraVolumes: + extraContainerVolumes: # Request a TLS certificate from the secret-operator - name: tls ephemeral: diff --git a/stacks/observability/jaeger.yaml b/stacks/observability/jaeger.yaml index 107f15f7..fa99b017 100644 --- a/stacks/observability/jaeger.yaml +++ b/stacks/observability/jaeger.yaml @@ -11,8 +11,9 @@ repo: # helm search repo jaegertracing/jaeger version: 3.4.1 # 1.53.0 options: - # labels: - # stackable.tech/vendor: Stackable + commonLabels: + stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} provisionDataStore: cassandra: false allInOne: diff --git a/stacks/observability/opentelemetry-operator.yaml b/stacks/observability/opentelemetry-operator.yaml index 1ddb7a1a..c93debf5 100644 --- a/stacks/observability/opentelemetry-operator.yaml +++ b/stacks/observability/opentelemetry-operator.yaml @@ -8,6 +8,9 @@ repo: # Find the newest chart at https://github.com/open-telemetry/opentelemetry-helm-charts/releases version: 0.99.0 # 0.138.0 options: + additionalLabels: + stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} admissionWebhooks: certManager: enabled: false diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index b75ccc44..b4b0da7b 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -134,6 +134,9 @@ stacks: - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/observability/opentelemetry-collector-sidecar.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/observability/opentelemetry-collector-deployment.yaml parameters: + - name: stackLabel + description: Stack label to add to various components + default: observability - name: grafanaAdminPassword description: Password of the Grafana admin user default: adminadmin From a02b92ae571efadd4439a11b8add54197db896f3 Mon Sep 17 00:00:00 2001 From: xeniape Date: Tue, 10 Mar 2026 16:59:35 +0100 Subject: [PATCH 14/22] tutorial-openldap in non-default ns and with demo labels --- stacks/authentication/openldap-tls.yaml | 2 +- stacks/stacks-v2.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stacks/authentication/openldap-tls.yaml b/stacks/authentication/openldap-tls.yaml index 4822ce1b..de35ae3c 100644 --- a/stacks/authentication/openldap-tls.yaml +++ b/stacks/authentication/openldap-tls.yaml @@ -10,7 +10,7 @@ spec: autoGenerate: true secret: name: openldap-tls-ca - namespace: default + namespace: {{ NAMESPACE }} --- apiVersion: apps/v1 kind: StatefulSet diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index b4b0da7b..1a97cd8a 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -538,7 +538,7 @@ stacks: - ldap manifests: - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/authentication/openldap-tls.yaml - supportedNamespaces: ["default"] + supportedNamespaces: [] resourceRequests: cpu: 1950m memory: 2182Mi From ed92e91f65fd124689e21db0b13f54b27757f1ce Mon Sep 17 00:00:00 2001 From: xeniape Date: Tue, 10 Mar 2026 18:21:16 +0100 Subject: [PATCH 15/22] openldap in non-default ns and with demo labels --- stacks/authentication/openldap-tls-authenticationclass.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/authentication/openldap-tls-authenticationclass.yaml b/stacks/authentication/openldap-tls-authenticationclass.yaml index 6a96c846..e99bda46 100644 --- a/stacks/authentication/openldap-tls-authenticationclass.yaml +++ b/stacks/authentication/openldap-tls-authenticationclass.yaml @@ -6,7 +6,7 @@ metadata: spec: provider: ldap: - hostname: openldap.default.svc.cluster.local + hostname: openldap.{{ NAMESPACE }}.svc.cluster.local searchBase: ou=users,dc=example,dc=org # The bind credentials are used so an application can bind to the LDAP server bindCredentials: From cf9e661b77f1fb27271d37a10a3e77ac6d2f7b07 Mon Sep 17 00:00:00 2001 From: xeniape Date: Tue, 10 Mar 2026 20:20:56 +0100 Subject: [PATCH 16/22] argocd-cd-git-ops demo in non-default ns and with demo labels --- .../applications/airflow-postgres.yaml | 2 ++ demos/argo-cd-git-ops/applications/airflow.yaml | 2 ++ demos/argo-cd-git-ops/applications/minio.yaml | 2 ++ .../sealed-airflow-postgres-credentials.yaml | 3 +++ .../manifests/airflow/airflow.yaml | 3 +++ .../airflow/sealed-airflow-credentials.yaml | 3 +++ .../airflow/sealed-airflow-minio-connection.yaml | 3 +++ demos/argo-cd-git-ops/manifests/minio/minio.yaml | 15 +++++++++++++++ .../manifests/minio/sealed-minio-credentials.yaml | 3 +++ demos/demos-v2.yaml | 3 +++ stacks/_templates/argo-cd.yaml | 4 ++++ .../applications/sealed-secrets.yaml | 2 ++ .../secrets/sealed-secrets-key.yaml | 3 +++ stacks/stacks-v2.yaml | 3 +++ 14 files changed, 51 insertions(+) diff --git a/demos/argo-cd-git-ops/applications/airflow-postgres.yaml b/demos/argo-cd-git-ops/applications/airflow-postgres.yaml index 08723cb4..3391bfa8 100644 --- a/demos/argo-cd-git-ops/applications/airflow-postgres.yaml +++ b/demos/argo-cd-git-ops/applications/airflow-postgres.yaml @@ -3,6 +3,8 @@ apiVersion: argoproj.io/v1alpha1 kind: Application metadata: name: airflow-postgres + finalizers: + - resources-finalizer.argocd.argoproj.io spec: project: airflow destination: diff --git a/demos/argo-cd-git-ops/applications/airflow.yaml b/demos/argo-cd-git-ops/applications/airflow.yaml index 276ede5e..52d4f46f 100644 --- a/demos/argo-cd-git-ops/applications/airflow.yaml +++ b/demos/argo-cd-git-ops/applications/airflow.yaml @@ -3,6 +3,8 @@ apiVersion: argoproj.io/v1alpha1 kind: Application metadata: name: airflow + finalizers: + - resources-finalizer.argocd.argoproj.io spec: project: airflow destination: diff --git a/demos/argo-cd-git-ops/applications/minio.yaml b/demos/argo-cd-git-ops/applications/minio.yaml index 12e220fa..61df2292 100644 --- a/demos/argo-cd-git-ops/applications/minio.yaml +++ b/demos/argo-cd-git-ops/applications/minio.yaml @@ -3,6 +3,8 @@ apiVersion: argoproj.io/v1alpha1 kind: Application metadata: name: minio + finalizers: + - resources-finalizer.argocd.argoproj.io spec: project: minio destination: diff --git a/demos/argo-cd-git-ops/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml b/demos/argo-cd-git-ops/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml index 51bd342f..7e69fd60 100644 --- a/demos/argo-cd-git-ops/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml +++ b/demos/argo-cd-git-ops/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml @@ -16,4 +16,7 @@ spec: creationTimestamp: null name: postgresql-credentials namespace: stackable-airflow + labels: + stackable.tech/vendor: Stackable + stackable.tech/demo: {{ demoLabel }} type: Opaque diff --git a/demos/argo-cd-git-ops/manifests/airflow/airflow.yaml b/demos/argo-cd-git-ops/manifests/airflow/airflow.yaml index 5c00aeed..69aa26fa 100644 --- a/demos/argo-cd-git-ops/manifests/airflow/airflow.yaml +++ b/demos/argo-cd-git-ops/manifests/airflow/airflow.yaml @@ -3,6 +3,9 @@ apiVersion: airflow.stackable.tech/v1alpha1 kind: AirflowCluster metadata: name: airflow + labels: + stackable.tech/vendor: Stackable + stackable.tech/demo: {{ demoLabel }} spec: image: productVersion: 3.1.6 diff --git a/demos/argo-cd-git-ops/manifests/airflow/sealed-airflow-credentials.yaml b/demos/argo-cd-git-ops/manifests/airflow/sealed-airflow-credentials.yaml index 9f763340..b5459962 100644 --- a/demos/argo-cd-git-ops/manifests/airflow/sealed-airflow-credentials.yaml +++ b/demos/argo-cd-git-ops/manifests/airflow/sealed-airflow-credentials.yaml @@ -21,4 +21,7 @@ spec: creationTimestamp: null name: airflow-credentials namespace: stackable-airflow + labels: + stackable.tech/vendor: Stackable + stackable.tech/demo: {{ demoLabel }} type: Opaque diff --git a/demos/argo-cd-git-ops/manifests/airflow/sealed-airflow-minio-connection.yaml b/demos/argo-cd-git-ops/manifests/airflow/sealed-airflow-minio-connection.yaml index 8e9f4f8d..fb395591 100644 --- a/demos/argo-cd-git-ops/manifests/airflow/sealed-airflow-minio-connection.yaml +++ b/demos/argo-cd-git-ops/manifests/airflow/sealed-airflow-minio-connection.yaml @@ -13,4 +13,7 @@ spec: creationTimestamp: null name: airflow-minio-connection namespace: stackable-airflow + labels: + stackable.tech/vendor: Stackable + stackable.tech/demo: {{ demoLabel }} type: Opaque diff --git a/demos/argo-cd-git-ops/manifests/minio/minio.yaml b/demos/argo-cd-git-ops/manifests/minio/minio.yaml index 63aa630e..50ad9dd9 100644 --- a/demos/argo-cd-git-ops/manifests/minio/minio.yaml +++ b/demos/argo-cd-git-ops/manifests/minio/minio.yaml @@ -4,6 +4,9 @@ apiVersion: v1 kind: ServiceAccount metadata: name: "minio-sa" + labels: + stackable.tech/vendor: Stackable + stackable.tech/demo: {{ demoLabel }} --- # Source: minio/templates/configmap.yaml apiVersion: v1 @@ -15,6 +18,8 @@ metadata: chart: minio-5.4.0 release: minio heritage: Helm + stackable.tech/vendor: Stackable + stackable.tech/demo: {{ demoLabel }} data: initialize: |- #!/bin/sh @@ -408,6 +413,8 @@ metadata: chart: minio-5.4.0 release: minio heritage: Helm + stackable.tech/vendor: Stackable + stackable.tech/demo: {{ demoLabel }} spec: accessModes: - "ReadWriteOnce" @@ -425,6 +432,8 @@ metadata: chart: minio-5.4.0 release: minio heritage: Helm + stackable.tech/vendor: Stackable + stackable.tech/demo: {{ demoLabel }} spec: type: NodePort externalTrafficPolicy: "Cluster" @@ -448,6 +457,8 @@ metadata: release: minio heritage: Helm monitoring: "true" + stackable.tech/vendor: Stackable + stackable.tech/demo: {{ demoLabel }} spec: type: NodePort externalTrafficPolicy: "Cluster" @@ -471,6 +482,7 @@ metadata: release: minio heritage: Helm stackable.tech/vendor: Stackable + stackable.tech/demo: {{ demoLabel }} spec: strategy: type: RollingUpdate @@ -489,6 +501,7 @@ spec: app: minio release: minio stackable.tech/vendor: Stackable + stackable.tech/demo: {{ demoLabel }} annotations: checksum/secrets: fa63e34a92c817c84057e2d452fa683e66462a57b0529388fb96a57e05f38e57 checksum/config: ebea49cc4c1bfbd1b156a58bf770a776ff87fe199f642d31c2816b5515112e72 @@ -583,6 +596,8 @@ metadata: chart: minio-5.4.0 release: minio heritage: Helm + stackable.tech/vendor: Stackable + stackable.tech/demo: {{ demoLabel }} annotations: "helm.sh/hook": post-install,post-upgrade "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation diff --git a/demos/argo-cd-git-ops/manifests/minio/sealed-minio-credentials.yaml b/demos/argo-cd-git-ops/manifests/minio/sealed-minio-credentials.yaml index 5a030480..022eb7d5 100644 --- a/demos/argo-cd-git-ops/manifests/minio/sealed-minio-credentials.yaml +++ b/demos/argo-cd-git-ops/manifests/minio/sealed-minio-credentials.yaml @@ -14,4 +14,7 @@ spec: creationTimestamp: null name: minio namespace: minio + labels: + stackable.tech/vendor: Stackable + stackable.tech/demo: {{ demoLabel }} type: Opaque diff --git a/demos/demos-v2.yaml b/demos/demos-v2.yaml index 9204b852..79efc605 100644 --- a/demos/demos-v2.yaml +++ b/demos/demos-v2.yaml @@ -33,6 +33,9 @@ demos: memory: 15Gi pvc: 15Gi parameters: + - name: demoLabel + description: Demo label to add to various components + default: argo-cd-git-ops - name: customGitUrl description: The URL to the Git repository (this can be adapted in case of forking the repository) to demonstrate syncing via Argo and Git. default: https://github.com/stackabletech/demos/ diff --git a/stacks/_templates/argo-cd.yaml b/stacks/_templates/argo-cd.yaml index 69ce0884..7590d9ac 100644 --- a/stacks/_templates/argo-cd.yaml +++ b/stacks/_templates/argo-cd.yaml @@ -6,6 +6,10 @@ repo: url: https://argoproj.github.io/argo-helm version: 9.0.5 # v3.1.9 options: + global: + additionalLabels: + stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} configs: secret: argocdServerAdminPassword: "{{ argocdAdminPassword }}" diff --git a/stacks/argo-cd-git-ops/applications/sealed-secrets.yaml b/stacks/argo-cd-git-ops/applications/sealed-secrets.yaml index 047b0ffe..350201d1 100644 --- a/stacks/argo-cd-git-ops/applications/sealed-secrets.yaml +++ b/stacks/argo-cd-git-ops/applications/sealed-secrets.yaml @@ -3,6 +3,8 @@ apiVersion: argoproj.io/v1alpha1 kind: Application metadata: name: sealed-secrets + finalizers: + - resources-finalizer.argocd.argoproj.io spec: project: default sources: diff --git a/stacks/argo-cd-git-ops/secrets/sealed-secrets-key.yaml b/stacks/argo-cd-git-ops/secrets/sealed-secrets-key.yaml index 7196e082..bc196947 100644 --- a/stacks/argo-cd-git-ops/secrets/sealed-secrets-key.yaml +++ b/stacks/argo-cd-git-ops/secrets/sealed-secrets-key.yaml @@ -3,6 +3,9 @@ apiVersion: v1 kind: Secret metadata: name: sealed-secrets-key + labels: + stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} stringData: # Generated on 2025/04/12 # openssl req -x509 -nodes -newkey rsa:4096 -keyout tls.key -out tls.crt -subj "/CN=sealed-secrets" -days 3650 diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index 1a97cd8a..5ca4ce3b 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -33,6 +33,9 @@ stacks: cpu: 5000m pvc: 5Gi parameters: + - name: stackLabel + description: Stack label to add to various components + default: argo-cd-git-ops - name: stackableReleaseVersion description: The Stackable release to be installed via Argo. This replaces the demo `stackableRelease` parameter. # TODO(@maltesander): Must be adapted in case of release ! From 5d737ad537fad3f49c6676ac2414e1efd2e8c01c Mon Sep 17 00:00:00 2001 From: xeniape Date: Tue, 10 Mar 2026 20:47:50 +0100 Subject: [PATCH 17/22] data-lakehouse-iceberg-trino-spark demo in non-default ns and with demo labels --- .../create-nifi-ingestion-job.yaml | 7 +++---- .../create-spark-ingestion-job.yaml | 9 +++++---- .../create-trino-tables.yaml | 2 +- .../load-test-data.yaml | 2 +- .../serviceaccount.yaml | 3 +-- .../setup-superset.yaml | 1 + demos/demos-v2.yaml | 6 +++++- .../_templates/minio-distributed-tls/rendered-chart.yaml | 2 +- stacks/_templates/minio-distributed-tls/values.yaml | 2 ++ .../s3-connection.yaml | 2 +- stacks/stacks-v2.yaml | 5 ++++- 11 files changed, 25 insertions(+), 16 deletions(-) diff --git a/demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml b/demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml index ab8d6eb1..dfe3cf19 100644 --- a/demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml +++ b/demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml @@ -17,9 +17,9 @@ spec: - -c - | echo 'Waiting for all nifi nodes to be ready' - kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/instance=nifi,app.kubernetes.io/name=nifi + kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/instance=nifi,app.kubernetes.io/name=nifi -n {{ NAMESPACE }} echo 'Waiting for all kafka brokers to be ready' - kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/instance=kafka,app.kubernetes.io/name=kafka + kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/instance=kafka,app.kubernetes.io/name=kafka -n {{ NAMESPACE }} containers: - name: create-nifi-ingestion-job image: oci.stackable.tech/sdp/testing-tools/nifi:0.3.0-stackable0.0.0-dev @@ -128,7 +128,6 @@ apiVersion: v1 kind: ConfigMap metadata: name: ingest-data-lake-iceberg-trino-spark-flow - namespace: default data: IngestDataLakeIcebergTrinoSpark.json: | - { "flowContents": { "identifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "instanceIdentifier": "8166cd44-0195-1000-ffff-ffffe3e76943", "name": "LakehouseKafkaIngest_raw", "comments": "", "position": { "x": -40.0, "y": -416.0 }, "processGroups": [], "remoteProcessGroups": [], "processors": [ { "identifier": "5e50831e-a2fd-34c9-af0a-924b16280705", "instanceIdentifier": "54f2ba99-3a49-3160-b8b9-45b8c0a6ac2a", "name": "Get station information", "comments": "", "position": { "x": 1701.0000007628792, "y": 273.999988448508 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://api.mobidata-bw.de/sharing/gbfs/v2/deer/station_information", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "false", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "10 m", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "PRIMARY", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "ba73d7c1-102f-3888-a573-5e6d2ec9e1f0", "instanceIdentifier": "816f11f1-0195-1000-ffff-ffffd698d6c2", "name": "Add station_uuid", "comments": "", "position": { "x": 936.0, "y": 1216.0 }, "type": "org.apache.nifi.processors.jolt.JoltTransformJSON", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-jolt-nar", "version": "2.6.0" }, "properties": { "Jolt Transform": "jolt-transform-chain", "Max String Length": "20 MB", "Transform Cache Size": "1", "Custom Transformation Class Name": null, "Custom Module Directory": null, "Pretty Print": "false", "Jolt Specification": "[\n {\n \"operation\": \"default\",\n \"spec\": {\n \"*\": {\n \"station_uuid\": \"${station_uuid}\"\n }\n }\n }\n]\n" }, "propertyDescriptors": { "Jolt Transform": { "name": "Jolt Transform", "displayName": "Jolt Transform", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Max String Length": { "name": "Max String Length", "displayName": "Max String Length", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transform Cache Size": { "name": "Transform Cache Size", "displayName": "Transform Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Transformation Class Name": { "name": "Custom Transformation Class Name", "displayName": "Custom Transformation Class Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Module Directory": { "name": "Custom Module Directory", "displayName": "Custom Module Directory", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "MULTIPLE", "resourceTypes": [ "DIRECTORY", "FILE" ] } }, "Pretty Print": { "name": "Pretty Print", "displayName": "Pretty Print", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Jolt Specification": { "name": "Jolt Specification", "displayName": "Jolt Specification", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "SINGLE", "resourceTypes": [ "TEXT", "FILE" ] } } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "577f603d-ee9c-3202-8139-d35d94c9c311", "instanceIdentifier": "331d7feb-96a3-3998-9a1e-5b23cca85c5f", "name": "SplitRecord", "comments": "", "position": { "x": 1309.0000007628792, "y": 537.999988448508 }, "type": "org.apache.nifi.processors.standard.SplitRecord", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Record Writer": "50e96367-7b84-3034-8dbf-6e02fa7c01df", "Record Reader": "c127588a-e95f-382f-afb1-33f9b279c38d", "Records Per Split": "1" }, "propertyDescriptors": { "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Records Per Split": { "name": "Records Per Split", "displayName": "Records Per Split", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "original", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "fc594301-ecc4-353b-8330-7aecb14e6422", "instanceIdentifier": "f049f0b7-46f9-332b-bce3-925167b05f37", "name": "Get station list", "comments": "", "position": { "x": 1309.0000007628792, "y": 273.999988448508 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations.json", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "false", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "15 m", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "PRIMARY", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "3a46b8c1-c2c6-3006-0000-000066d6dbdd", "instanceIdentifier": "9e653a1f-9897-3124-8aa1-83321219f2fc", "name": "Extract data attribute", "comments": "", "position": { "x": 2104.0, "y": 536.0 }, "type": "org.apache.nifi.processors.jolt.JoltTransformJSON", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-jolt-nar", "version": "2.6.0" }, "properties": { "Jolt Transform": "jolt-transform-chain", "Max String Length": "20 MB", "Transform Cache Size": "1", "Custom Transformation Class Name": null, "Custom Module Directory": null, "Pretty Print": "false", "Jolt Specification": "[\n {\n \"operation\": \"shift\",\n \"spec\": {\n \"data\": {\n \"*\": \"\"\n }\n }\n }\n]\n" }, "propertyDescriptors": { "Jolt Transform": { "name": "Jolt Transform", "displayName": "Jolt Transform", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Max String Length": { "name": "Max String Length", "displayName": "Max String Length", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transform Cache Size": { "name": "Transform Cache Size", "displayName": "Transform Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Transformation Class Name": { "name": "Custom Transformation Class Name", "displayName": "Custom Transformation Class Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Module Directory": { "name": "Custom Module Directory", "displayName": "Custom Module Directory", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "MULTIPLE", "resourceTypes": [ "DIRECTORY", "FILE" ] } }, "Pretty Print": { "name": "Pretty Print", "displayName": "Pretty Print", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Jolt Specification": { "name": "Jolt Specification", "displayName": "Jolt Specification", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "SINGLE", "resourceTypes": [ "TEXT", "FILE" ] } } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "dd4690f7-ae51-36bb-0000-0000325e1eed", "instanceIdentifier": "664aded4-922a-3da4-a584-7d5e43361201", "name": "Produce station status records", "comments": "", "position": { "x": 2104.0, "y": 752.0 }, "type": "org.apache.nifi.kafka.processors.PublishKafka", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-nar", "version": "2.6.0" }, "properties": { "compression.type": "none", "acks": "all", "Message Demarcator": null, "Kafka Key Attribute Encoding": "utf-8", "Record Reader": "c127588a-e95f-382f-afb1-33f9b279c38d", "Record Metadata Strategy": "FROM_PROPERTIES", "Transactional ID Prefix": null, "Header Encoding": "UTF-8", "max.request.size": "1 MB", "Topic Name": "shared_bikes_station_status", "Kafka Key": null, "partition": null, "Kafka Connection Service": "a7cc755f-2210-3763-889e-24c3e0b0684d", "Publish Strategy": "USE_VALUE", "Record Key Writer": null, "Failure Strategy": "Route to Failure", "partitioner.class": "org.apache.kafka.clients.producer.internals.DefaultPartitioner", "Record Writer": "50e96367-7b84-3034-8dbf-6e02fa7c01df", "Transactions Enabled": "false", "Message Key Field": null, "FlowFile Attribute Header Pattern": null }, "propertyDescriptors": { "compression.type": { "name": "compression.type", "displayName": "Compression Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "acks": { "name": "acks", "displayName": "Delivery Guarantee", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Demarcator": { "name": "Message Demarcator", "displayName": "Message Demarcator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key Attribute Encoding": { "name": "Kafka Key Attribute Encoding", "displayName": "Kafka Key Attribute Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Metadata Strategy": { "name": "Record Metadata Strategy", "displayName": "Record Metadata Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transactional ID Prefix": { "name": "Transactional ID Prefix", "displayName": "Transactional ID Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Header Encoding": { "name": "Header Encoding", "displayName": "Header Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.request.size": { "name": "max.request.size", "displayName": "Max Request Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Topic Name": { "name": "Topic Name", "displayName": "Topic Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key": { "name": "Kafka Key", "displayName": "Kafka Key", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partition": { "name": "partition", "displayName": "Partition", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Connection Service": { "name": "Kafka Connection Service", "displayName": "Kafka Connection Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Publish Strategy": { "name": "Publish Strategy", "displayName": "Publish Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Key Writer": { "name": "Record Key Writer", "displayName": "Record Key Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Failure Strategy": { "name": "Failure Strategy", "displayName": "Failure Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partitioner.class": { "name": "partitioner.class", "displayName": "Partitioner Class", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Transactions Enabled": { "name": "Transactions Enabled", "displayName": "Transactions Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Key Field": { "name": "Message Key Field", "displayName": "Message Key Field", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "FlowFile Attribute Header Pattern": { "name": "FlowFile Attribute Header Pattern", "displayName": "FlowFile Attribute Header Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "success", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "90d02439-1516-3c8a-ffff-ffff8b2ffb20", "instanceIdentifier": "3a5f9f2c-bafa-3ae3-bc1b-2e8f1dbea59e", "name": "Produce station information records", "comments": "", "position": { "x": 1704.0, "y": 752.0 }, "type": "org.apache.nifi.kafka.processors.PublishKafka", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-nar", "version": "2.6.0" }, "properties": { "compression.type": "none", "acks": "all", "Message Demarcator": null, "Kafka Key Attribute Encoding": "utf-8", "Record Reader": "c127588a-e95f-382f-afb1-33f9b279c38d", "Record Metadata Strategy": "FROM_PROPERTIES", "Transactional ID Prefix": null, "Header Encoding": "UTF-8", "max.request.size": "1 MB", "Topic Name": "shared_bikes_station_information", "Kafka Key": null, "partition": null, "Kafka Connection Service": "a7cc755f-2210-3763-889e-24c3e0b0684d", "Publish Strategy": "USE_VALUE", "Record Key Writer": null, "Failure Strategy": "Route to Failure", "partitioner.class": "org.apache.kafka.clients.producer.internals.DefaultPartitioner", "Record Writer": "50e96367-7b84-3034-8dbf-6e02fa7c01df", "Transactions Enabled": "false", "Message Key Field": null, "FlowFile Attribute Header Pattern": null }, "propertyDescriptors": { "compression.type": { "name": "compression.type", "displayName": "Compression Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "acks": { "name": "acks", "displayName": "Delivery Guarantee", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Demarcator": { "name": "Message Demarcator", "displayName": "Message Demarcator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key Attribute Encoding": { "name": "Kafka Key Attribute Encoding", "displayName": "Kafka Key Attribute Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Metadata Strategy": { "name": "Record Metadata Strategy", "displayName": "Record Metadata Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transactional ID Prefix": { "name": "Transactional ID Prefix", "displayName": "Transactional ID Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Header Encoding": { "name": "Header Encoding", "displayName": "Header Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.request.size": { "name": "max.request.size", "displayName": "Max Request Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Topic Name": { "name": "Topic Name", "displayName": "Topic Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key": { "name": "Kafka Key", "displayName": "Kafka Key", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partition": { "name": "partition", "displayName": "Partition", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Connection Service": { "name": "Kafka Connection Service", "displayName": "Kafka Connection Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Publish Strategy": { "name": "Publish Strategy", "displayName": "Publish Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Key Writer": { "name": "Record Key Writer", "displayName": "Record Key Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Failure Strategy": { "name": "Failure Strategy", "displayName": "Failure Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partitioner.class": { "name": "partitioner.class", "displayName": "Partitioner Class", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Transactions Enabled": { "name": "Transactions Enabled", "displayName": "Transactions Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Key Field": { "name": "Message Key Field", "displayName": "Message Key Field", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "FlowFile Attribute Header Pattern": { "name": "FlowFile Attribute Header Pattern", "displayName": "FlowFile Attribute Header Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "success", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "056f4faa-a4dc-3024-ffff-ffffb73ef3a0", "instanceIdentifier": "7b0b12f3-669e-32c4-b866-dea8a4a6f34a", "name": "Extract data attribute", "comments": "", "position": { "x": 2496.0, "y": 536.0 }, "type": "org.apache.nifi.processors.jolt.JoltTransformJSON", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-jolt-nar", "version": "2.6.0" }, "properties": { "Jolt Transform": "jolt-transform-chain", "Max String Length": "20 MB", "Transform Cache Size": "1", "Custom Transformation Class Name": null, "Custom Module Directory": null, "Pretty Print": "false", "Jolt Specification": "[\n {\n \"operation\": \"shift\",\n \"spec\": {\n \"data\": {\n \"*\": \"\"\n }\n }\n }\n]\n" }, "propertyDescriptors": { "Jolt Transform": { "name": "Jolt Transform", "displayName": "Jolt Transform", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Max String Length": { "name": "Max String Length", "displayName": "Max String Length", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transform Cache Size": { "name": "Transform Cache Size", "displayName": "Transform Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Transformation Class Name": { "name": "Custom Transformation Class Name", "displayName": "Custom Transformation Class Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Module Directory": { "name": "Custom Module Directory", "displayName": "Custom Module Directory", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "MULTIPLE", "resourceTypes": [ "DIRECTORY", "FILE" ] } }, "Pretty Print": { "name": "Pretty Print", "displayName": "Pretty Print", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Jolt Specification": { "name": "Jolt Specification", "displayName": "Jolt Specification", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "SINGLE", "resourceTypes": [ "TEXT", "FILE" ] } } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "42346af3-5793-3b8c-ffff-ffffc5c5139d", "instanceIdentifier": "f9f95954-77fc-3f16-9b94-54ca68090b21", "name": "Add station_uuid", "comments": "", "position": { "x": 1312.0, "y": 1216.0 }, "type": "org.apache.nifi.processors.jolt.JoltTransformJSON", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-jolt-nar", "version": "2.6.0" }, "properties": { "Jolt Transform": "jolt-transform-chain", "Max String Length": "20 MB", "Transform Cache Size": "1", "Custom Transformation Class Name": null, "Custom Module Directory": null, "Pretty Print": "false", "Jolt Specification": "[\n {\n \"operation\": \"default\",\n \"spec\": {\n \"*\": {\n \"station_uuid\": \"${station_uuid}\"\n }\n }\n }\n]\n" }, "propertyDescriptors": { "Jolt Transform": { "name": "Jolt Transform", "displayName": "Jolt Transform", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Max String Length": { "name": "Max String Length", "displayName": "Max String Length", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transform Cache Size": { "name": "Transform Cache Size", "displayName": "Transform Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Transformation Class Name": { "name": "Custom Transformation Class Name", "displayName": "Custom Transformation Class Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Module Directory": { "name": "Custom Module Directory", "displayName": "Custom Module Directory", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "MULTIPLE", "resourceTypes": [ "DIRECTORY", "FILE" ] } }, "Pretty Print": { "name": "Pretty Print", "displayName": "Pretty Print", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Jolt Specification": { "name": "Jolt Specification", "displayName": "Jolt Specification", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "SINGLE", "resourceTypes": [ "TEXT", "FILE" ] } } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "1a0aea35-946c-35de-8811-549faeb003ee", "instanceIdentifier": "81675904-0195-1000-ffff-ffffbbbb9cfe", "name": "Produce station records", "comments": "", "position": { "x": 560.0, "y": 536.0 }, "type": "org.apache.nifi.kafka.processors.PublishKafka", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-nar", "version": "2.6.0" }, "properties": { "compression.type": "none", "acks": "all", "Message Demarcator": null, "Kafka Key Attribute Encoding": "utf-8", "Record Reader": "c127588a-e95f-382f-afb1-33f9b279c38d", "Record Metadata Strategy": "FROM_PROPERTIES", "Transactional ID Prefix": null, "Header Encoding": "UTF-8", "max.request.size": "1 MB", "Topic Name": "water_levels_stations", "Kafka Key": null, "partition": null, "Kafka Connection Service": "a7cc755f-2210-3763-889e-24c3e0b0684d", "Publish Strategy": "USE_VALUE", "Record Key Writer": null, "Failure Strategy": "Route to Failure", "partitioner.class": "org.apache.kafka.clients.producer.internals.DefaultPartitioner", "Record Writer": "50e96367-7b84-3034-8dbf-6e02fa7c01df", "Transactions Enabled": "false", "Message Key Field": null, "FlowFile Attribute Header Pattern": null }, "propertyDescriptors": { "compression.type": { "name": "compression.type", "displayName": "Compression Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "acks": { "name": "acks", "displayName": "Delivery Guarantee", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Demarcator": { "name": "Message Demarcator", "displayName": "Message Demarcator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key Attribute Encoding": { "name": "Kafka Key Attribute Encoding", "displayName": "Kafka Key Attribute Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Metadata Strategy": { "name": "Record Metadata Strategy", "displayName": "Record Metadata Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transactional ID Prefix": { "name": "Transactional ID Prefix", "displayName": "Transactional ID Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Header Encoding": { "name": "Header Encoding", "displayName": "Header Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.request.size": { "name": "max.request.size", "displayName": "Max Request Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Topic Name": { "name": "Topic Name", "displayName": "Topic Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key": { "name": "Kafka Key", "displayName": "Kafka Key", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partition": { "name": "partition", "displayName": "Partition", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Connection Service": { "name": "Kafka Connection Service", "displayName": "Kafka Connection Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Publish Strategy": { "name": "Publish Strategy", "displayName": "Publish Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Key Writer": { "name": "Record Key Writer", "displayName": "Record Key Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Failure Strategy": { "name": "Failure Strategy", "displayName": "Failure Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partitioner.class": { "name": "partitioner.class", "displayName": "Partitioner Class", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Transactions Enabled": { "name": "Transactions Enabled", "displayName": "Transactions Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Key Field": { "name": "Message Key Field", "displayName": "Message Key Field", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "FlowFile Attribute Header Pattern": { "name": "FlowFile Attribute Header Pattern", "displayName": "FlowFile Attribute Header Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "success", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "364e250b-f224-3e12-b4cf-c782c9a06bbe", "instanceIdentifier": "143b2ea1-f9cc-3140-afd1-426a846dd9d0", "name": "Get free bike status", "comments": "", "position": { "x": 2501.000000762879, "y": 273.999988448508 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://api.mobidata-bw.de/sharing/gbfs/v2/deer/free_bike_status", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "false", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "1 m", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "PRIMARY", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "26a592be-7444-337c-b8f9-3a29013cb760", "instanceIdentifier": "0ea082f4-3f0d-38db-bd4b-51feb5428cc6", "name": "Get station status", "comments": "", "position": { "x": 2101.000000762879, "y": 273.999988448508 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://api.mobidata-bw.de/sharing/gbfs/v2/deer/station_status", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "false", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "1 m", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "PRIMARY", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "d9bacb7c-10df-3044-ffff-ffffe7a604af", "instanceIdentifier": "8c7c157b-6f95-399c-83f7-54a79ea11078", "name": "Produce bike status records", "comments": "", "position": { "x": 2504.0, "y": 752.0 }, "type": "org.apache.nifi.kafka.processors.PublishKafka", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-nar", "version": "2.6.0" }, "properties": { "compression.type": "none", "acks": "all", "Message Demarcator": null, "Kafka Key Attribute Encoding": "utf-8", "Record Reader": "c127588a-e95f-382f-afb1-33f9b279c38d", "Record Metadata Strategy": "FROM_PROPERTIES", "Transactional ID Prefix": null, "Header Encoding": "UTF-8", "max.request.size": "1 MB", "Topic Name": "shared_bikes_bike_status", "Kafka Key": null, "partition": null, "Kafka Connection Service": "a7cc755f-2210-3763-889e-24c3e0b0684d", "Publish Strategy": "USE_VALUE", "Record Key Writer": null, "Failure Strategy": "Route to Failure", "partitioner.class": "org.apache.kafka.clients.producer.internals.DefaultPartitioner", "Record Writer": "50e96367-7b84-3034-8dbf-6e02fa7c01df", "Transactions Enabled": "false", "Message Key Field": null, "FlowFile Attribute Header Pattern": null }, "propertyDescriptors": { "compression.type": { "name": "compression.type", "displayName": "Compression Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "acks": { "name": "acks", "displayName": "Delivery Guarantee", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Demarcator": { "name": "Message Demarcator", "displayName": "Message Demarcator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key Attribute Encoding": { "name": "Kafka Key Attribute Encoding", "displayName": "Kafka Key Attribute Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Metadata Strategy": { "name": "Record Metadata Strategy", "displayName": "Record Metadata Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transactional ID Prefix": { "name": "Transactional ID Prefix", "displayName": "Transactional ID Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Header Encoding": { "name": "Header Encoding", "displayName": "Header Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.request.size": { "name": "max.request.size", "displayName": "Max Request Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Topic Name": { "name": "Topic Name", "displayName": "Topic Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key": { "name": "Kafka Key", "displayName": "Kafka Key", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partition": { "name": "partition", "displayName": "Partition", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Connection Service": { "name": "Kafka Connection Service", "displayName": "Kafka Connection Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Publish Strategy": { "name": "Publish Strategy", "displayName": "Publish Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Key Writer": { "name": "Record Key Writer", "displayName": "Record Key Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Failure Strategy": { "name": "Failure Strategy", "displayName": "Failure Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partitioner.class": { "name": "partitioner.class", "displayName": "Partitioner Class", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Transactions Enabled": { "name": "Transactions Enabled", "displayName": "Transactions Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Key Field": { "name": "Message Key Field", "displayName": "Message Key Field", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "FlowFile Attribute Header Pattern": { "name": "FlowFile Attribute Header Pattern", "displayName": "FlowFile Attribute Header Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "success", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "00bcf948-7bef-35b6-8a6d-e641194b8189", "instanceIdentifier": "d92196ba-38f9-3b77-b9a4-4e0dfe564d3f", "name": "Get station list", "comments": "", "position": { "x": 557.000000762879, "y": 273.999988448508 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations.json", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "false", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "10 m", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "PRIMARY", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "98c83f54-9f99-3d95-88b0-ae91140707c9", "instanceIdentifier": "098befce-068e-3d63-8586-1a7309e40e58", "name": "Get station list", "comments": "", "position": { "x": 933.000000762879, "y": 273.999988448508 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations.json", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "false", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "365000 days", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "PRIMARY", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "405f050c-5005-3b96-ffff-ffffbc8a379e", "instanceIdentifier": "3dd55349-9324-36ea-a6a0-d76280b79609", "name": "Extract data attribute", "comments": "", "position": { "x": 1704.0, "y": 544.0 }, "type": "org.apache.nifi.processors.jolt.JoltTransformJSON", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-jolt-nar", "version": "2.6.0" }, "properties": { "Jolt Transform": "jolt-transform-chain", "Max String Length": "20 MB", "Transform Cache Size": "1", "Custom Transformation Class Name": null, "Custom Module Directory": null, "Pretty Print": "false", "Jolt Specification": "[\n {\n \"operation\": \"shift\",\n \"spec\": {\n \"data\": {\n \"*\": \"\"\n }\n }\n }\n]\n" }, "propertyDescriptors": { "Jolt Transform": { "name": "Jolt Transform", "displayName": "Jolt Transform", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Max String Length": { "name": "Max String Length", "displayName": "Max String Length", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transform Cache Size": { "name": "Transform Cache Size", "displayName": "Transform Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Transformation Class Name": { "name": "Custom Transformation Class Name", "displayName": "Custom Transformation Class Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Module Directory": { "name": "Custom Module Directory", "displayName": "Custom Module Directory", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "MULTIPLE", "resourceTypes": [ "DIRECTORY", "FILE" ] } }, "Pretty Print": { "name": "Pretty Print", "displayName": "Pretty Print", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Jolt Specification": { "name": "Jolt Specification", "displayName": "Jolt Specification", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "SINGLE", "resourceTypes": [ "TEXT", "FILE" ] } } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "a1453322-1c3f-3150-a5be-0330c98144d9", "instanceIdentifier": "be5c2f57-09fd-3608-8c9c-9779e52c0ce8", "name": "Extract station_uuid", "comments": "", "position": { "x": 933.000000762879, "y": 753.999988448508 }, "type": "org.apache.nifi.processors.standard.EvaluateJsonPath", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Destination": "flowfile-attribute", "Max String Length": "20 MB", "Return Type": "auto-detect", "Null Value Representation": "empty string", "station_uuid": "$.uuid", "Path Not Found Behavior": "ignore" }, "propertyDescriptors": { "Destination": { "name": "Destination", "displayName": "Destination", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Max String Length": { "name": "Max String Length", "displayName": "Max String Length", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Return Type": { "name": "Return Type", "displayName": "Return Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Null Value Representation": { "name": "Null Value Representation", "displayName": "Null Value Representation", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "station_uuid": { "name": "station_uuid", "displayName": "station_uuid", "identifiesControllerService": false, "sensitive": false, "dynamic": true }, "Path Not Found Behavior": { "name": "Path Not Found Behavior", "displayName": "Path Not Found Behavior", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "failure", "unmatched" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "de71001e-de69-32c2-82b9-6f008665b252", "instanceIdentifier": "df138016-7764-3ad6-ac10-9f57e657461d", "name": "Get 30 minutes of historic data", "comments": "", "position": { "x": 1309.0000007628792, "y": 993.999988448508 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations/${station_uuid}/W/measurements.json?start=PT30M", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "false", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "1 s", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "7133f561-971a-35b6-abff-9892e06b4bbe", "instanceIdentifier": "fd7775f9-0f65-361a-bcb9-71668ae1a663", "name": "Get 30 days historic data", "comments": "", "position": { "x": 933.000000762879, "y": 993.999988448508 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations/${station_uuid}/W/measurements.json?start=P30D", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "false", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "0640256b-010a-3e69-b961-680e63ec85e2", "instanceIdentifier": "71fee077-9f08-3916-b189-f933b8f13619", "name": "Extract station_uuid", "comments": "", "position": { "x": 1309.0000007628792, "y": 753.999988448508 }, "type": "org.apache.nifi.processors.standard.EvaluateJsonPath", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Destination": "flowfile-attribute", "Max String Length": "20 MB", "Return Type": "auto-detect", "Null Value Representation": "empty string", "station_uuid": "$.uuid", "Path Not Found Behavior": "ignore" }, "propertyDescriptors": { "Destination": { "name": "Destination", "displayName": "Destination", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Max String Length": { "name": "Max String Length", "displayName": "Max String Length", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Return Type": { "name": "Return Type", "displayName": "Return Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Null Value Representation": { "name": "Null Value Representation", "displayName": "Null Value Representation", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "station_uuid": { "name": "station_uuid", "displayName": "station_uuid", "identifiesControllerService": false, "sensitive": false, "dynamic": true }, "Path Not Found Behavior": { "name": "Path Not Found Behavior", "displayName": "Path Not Found Behavior", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "failure", "unmatched" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "99d1a93c-6332-3e77-0000-0000527115be", "instanceIdentifier": "839f75a2-3a32-38b7-8c36-50498002e9c1", "name": "PublishKafka", "comments": "", "position": { "x": 936.0, "y": 1432.0 }, "type": "org.apache.nifi.kafka.processors.PublishKafka", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-nar", "version": "2.6.0" }, "properties": { "compression.type": "snappy", "acks": "all", "Message Demarcator": null, "Kafka Key Attribute Encoding": "utf-8", "Record Reader": "c127588a-e95f-382f-afb1-33f9b279c38d", "Record Metadata Strategy": "FROM_PROPERTIES", "Transactional ID Prefix": null, "Header Encoding": "UTF-8", "max.request.size": "1 MB", "Topic Name": "water_levels_measurements", "Kafka Key": null, "partition": null, "Kafka Connection Service": "a7cc755f-2210-3763-889e-24c3e0b0684d", "Publish Strategy": "USE_VALUE", "Record Key Writer": null, "Failure Strategy": "Route to Failure", "partitioner.class": "org.apache.kafka.clients.producer.internals.DefaultPartitioner", "Record Writer": "50e96367-7b84-3034-8dbf-6e02fa7c01df", "Transactions Enabled": "false", "Message Key Field": null, "FlowFile Attribute Header Pattern": null }, "propertyDescriptors": { "compression.type": { "name": "compression.type", "displayName": "Compression Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "acks": { "name": "acks", "displayName": "Delivery Guarantee", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Demarcator": { "name": "Message Demarcator", "displayName": "Message Demarcator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key Attribute Encoding": { "name": "Kafka Key Attribute Encoding", "displayName": "Kafka Key Attribute Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Metadata Strategy": { "name": "Record Metadata Strategy", "displayName": "Record Metadata Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transactional ID Prefix": { "name": "Transactional ID Prefix", "displayName": "Transactional ID Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Header Encoding": { "name": "Header Encoding", "displayName": "Header Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.request.size": { "name": "max.request.size", "displayName": "Max Request Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Topic Name": { "name": "Topic Name", "displayName": "Topic Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key": { "name": "Kafka Key", "displayName": "Kafka Key", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partition": { "name": "partition", "displayName": "Partition", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Connection Service": { "name": "Kafka Connection Service", "displayName": "Kafka Connection Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Publish Strategy": { "name": "Publish Strategy", "displayName": "Publish Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Key Writer": { "name": "Record Key Writer", "displayName": "Record Key Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Failure Strategy": { "name": "Failure Strategy", "displayName": "Failure Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partitioner.class": { "name": "partitioner.class", "displayName": "Partitioner Class", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Transactions Enabled": { "name": "Transactions Enabled", "displayName": "Transactions Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Key Field": { "name": "Message Key Field", "displayName": "Message Key Field", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "FlowFile Attribute Header Pattern": { "name": "FlowFile Attribute Header Pattern", "displayName": "FlowFile Attribute Header Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "success", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "762fb4f3-201b-3c92-ffff-ffff8e7e57fd", "instanceIdentifier": "3d7eb954-e054-3324-aea1-5aa7a03dc120", "name": "PublishKafka", "comments": "", "position": { "x": 1312.0, "y": 1432.0 }, "type": "org.apache.nifi.kafka.processors.PublishKafka", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-nar", "version": "2.6.0" }, "properties": { "compression.type": "snappy", "acks": "all", "Message Demarcator": null, "Kafka Key Attribute Encoding": "utf-8", "Record Reader": "c127588a-e95f-382f-afb1-33f9b279c38d", "Record Metadata Strategy": "FROM_PROPERTIES", "Transactional ID Prefix": null, "Header Encoding": "UTF-8", "max.request.size": "1 MB", "Topic Name": "water_levels_measurements", "Kafka Key": null, "partition": null, "Kafka Connection Service": "a7cc755f-2210-3763-889e-24c3e0b0684d", "Publish Strategy": "USE_VALUE", "Record Key Writer": null, "Failure Strategy": "Route to Failure", "partitioner.class": "org.apache.kafka.clients.producer.internals.DefaultPartitioner", "Record Writer": "50e96367-7b84-3034-8dbf-6e02fa7c01df", "Transactions Enabled": "false", "Message Key Field": null, "FlowFile Attribute Header Pattern": null }, "propertyDescriptors": { "compression.type": { "name": "compression.type", "displayName": "Compression Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "acks": { "name": "acks", "displayName": "Delivery Guarantee", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Demarcator": { "name": "Message Demarcator", "displayName": "Message Demarcator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key Attribute Encoding": { "name": "Kafka Key Attribute Encoding", "displayName": "Kafka Key Attribute Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Metadata Strategy": { "name": "Record Metadata Strategy", "displayName": "Record Metadata Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transactional ID Prefix": { "name": "Transactional ID Prefix", "displayName": "Transactional ID Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Header Encoding": { "name": "Header Encoding", "displayName": "Header Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.request.size": { "name": "max.request.size", "displayName": "Max Request Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Topic Name": { "name": "Topic Name", "displayName": "Topic Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key": { "name": "Kafka Key", "displayName": "Kafka Key", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partition": { "name": "partition", "displayName": "Partition", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Connection Service": { "name": "Kafka Connection Service", "displayName": "Kafka Connection Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Publish Strategy": { "name": "Publish Strategy", "displayName": "Publish Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Key Writer": { "name": "Record Key Writer", "displayName": "Record Key Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Failure Strategy": { "name": "Failure Strategy", "displayName": "Failure Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partitioner.class": { "name": "partitioner.class", "displayName": "Partitioner Class", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Transactions Enabled": { "name": "Transactions Enabled", "displayName": "Transactions Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Key Field": { "name": "Message Key Field", "displayName": "Message Key Field", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "FlowFile Attribute Header Pattern": { "name": "FlowFile Attribute Header Pattern", "displayName": "FlowFile Attribute Header Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "success", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "c6929ff2-ea95-3d79-a483-3ffa75371ee4", "instanceIdentifier": "78b9af30-7922-3dd6-91d4-f88d7f984ec6", "name": "SplitRecord", "comments": "", "position": { "x": 933.000000762879, "y": 537.999988448508 }, "type": "org.apache.nifi.processors.standard.SplitRecord", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Record Writer": "50e96367-7b84-3034-8dbf-6e02fa7c01df", "Record Reader": "c127588a-e95f-382f-afb1-33f9b279c38d", "Records Per Split": "1" }, "propertyDescriptors": { "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Records Per Split": { "name": "Records Per Split", "displayName": "Records Per Split", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "original", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" } ], "inputPorts": [], "outputPorts": [], "connections": [ { "identifier": "9552346e-6dd7-3991-99f5-479e87ebb4b6", "instanceIdentifier": "1bfdc782-8410-3e9d-a004-7d21c43fa5bb", "name": "", "source": { "id": "26a592be-7444-337c-b8f9-3a29013cb760", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Get station status", "comments": "", "instanceIdentifier": "0ea082f4-3f0d-38db-bd4b-51feb5428cc6" }, "destination": { "id": "3a46b8c1-c2c6-3006-0000-000066d6dbdd", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Extract data attribute", "comments": "", "instanceIdentifier": "9e653a1f-9897-3124-8aa1-83321219f2fc" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 10000, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "2c1c00a7-bf5d-3937-8fb3-8ee54b5117e3", "instanceIdentifier": "eb0cef96-08d3-39de-8518-cf136ccd5871", "name": "", "source": { "id": "5e50831e-a2fd-34c9-af0a-924b16280705", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Get station information", "comments": "", "instanceIdentifier": "54f2ba99-3a49-3160-b8b9-45b8c0a6ac2a" }, "destination": { "id": "405f050c-5005-3b96-ffff-ffffbc8a379e", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Extract data attribute", "comments": "", "instanceIdentifier": "3dd55349-9324-36ea-a6a0-d76280b79609" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 10000, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "e7f33855-2961-3414-8c28-3056fcdac571", "instanceIdentifier": "dc784d10-a696-3f10-94b8-f6bb1fae7235", "name": "", "source": { "id": "00bcf948-7bef-35b6-8a6d-e641194b8189", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Get station list", "comments": "", "instanceIdentifier": "d92196ba-38f9-3b77-b9a4-4e0dfe564d3f" }, "destination": { "id": "1a0aea35-946c-35de-8811-549faeb003ee", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Produce station records", "comments": "", "instanceIdentifier": "81675904-0195-1000-ffff-ffffbbbb9cfe" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 10000, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "f1255a8c-06e1-310c-bd02-5e34b218a02f", "instanceIdentifier": "8172e290-0195-1000-0000-0000188da43e", "name": "", "source": { "id": "056f4faa-a4dc-3024-ffff-ffffb73ef3a0", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Extract data attribute", "comments": "", "instanceIdentifier": "7b0b12f3-669e-32c4-b866-dea8a4a6f34a" }, "destination": { "id": "d9bacb7c-10df-3044-ffff-ffffe7a604af", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Produce bike status records", "comments": "", "instanceIdentifier": "8c7c157b-6f95-399c-83f7-54a79ea11078" }, "labelIndex": 0, "zIndex": 0, "selectedRelationships": [ "success" ], "backPressureObjectThreshold": 10000, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "ae09a6b6-1dcd-35bc-82ec-62171973c61b", "instanceIdentifier": "a742154d-a24c-3bb8-9f65-cbc95810a284", "name": "", "source": { "id": "de71001e-de69-32c2-82b9-6f008665b252", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Get 30 minutes of historic data", "comments": "", "instanceIdentifier": "df138016-7764-3ad6-ac10-9f57e657461d" }, "destination": { "id": "42346af3-5793-3b8c-ffff-ffffc5c5139d", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Add station_uuid", "comments": "", "instanceIdentifier": "f9f95954-77fc-3f16-9b94-54ca68090b21" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 10, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "709e7c78-6123-386a-a0ce-5a7394c4d39e", "instanceIdentifier": "9ab4b88c-e4c3-3303-88d1-18c8b5f00817", "name": "", "source": { "id": "fc594301-ecc4-353b-8330-7aecb14e6422", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Get station list", "comments": "", "instanceIdentifier": "f049f0b7-46f9-332b-bce3-925167b05f37" }, "destination": { "id": "577f603d-ee9c-3202-8139-d35d94c9c311", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "SplitRecord", "comments": "", "instanceIdentifier": "331d7feb-96a3-3998-9a1e-5b23cca85c5f" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 1, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "e4d5da00-e103-3166-8381-d4f2533c7a9d", "instanceIdentifier": "a559d40e-de21-3d84-a6e8-1abcfe13601a", "name": "", "source": { "id": "7133f561-971a-35b6-abff-9892e06b4bbe", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Get 30 days historic data", "comments": "", "instanceIdentifier": "fd7775f9-0f65-361a-bcb9-71668ae1a663" }, "destination": { "id": "ba73d7c1-102f-3888-a573-5e6d2ec9e1f0", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Add station_uuid", "comments": "", "instanceIdentifier": "816f11f1-0195-1000-ffff-ffffd698d6c2" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 10, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "fd7e3f54-ea73-3f6b-9700-33442d5ee235", "instanceIdentifier": "72965aa1-df12-31a6-9eb8-ac5d006406e0", "name": "", "source": { "id": "577f603d-ee9c-3202-8139-d35d94c9c311", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "SplitRecord", "comments": "", "instanceIdentifier": "331d7feb-96a3-3998-9a1e-5b23cca85c5f" }, "destination": { "id": "0640256b-010a-3e69-b961-680e63ec85e2", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Extract station_uuid", "comments": "", "instanceIdentifier": "71fee077-9f08-3916-b189-f933b8f13619" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "splits" ], "backPressureObjectThreshold": 10, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "ROUND_ROBIN", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "3bec8550-723b-3b2c-89c9-d57298a1ab03", "instanceIdentifier": "816ffee3-0195-1000-ffff-ffffd4be0c57", "name": "", "source": { "id": "ba73d7c1-102f-3888-a573-5e6d2ec9e1f0", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Add station_uuid", "comments": "", "instanceIdentifier": "816f11f1-0195-1000-ffff-ffffd698d6c2" }, "destination": { "id": "99d1a93c-6332-3e77-0000-0000527115be", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "PublishKafka", "comments": "", "instanceIdentifier": "839f75a2-3a32-38b7-8c36-50498002e9c1" }, "labelIndex": 0, "zIndex": 0, "selectedRelationships": [ "success" ], "backPressureObjectThreshold": 20, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "14af6270-d836-3045-a5ca-1e699f1da8ee", "instanceIdentifier": "81729813-0195-1000-0000-0000137d67da", "name": "", "source": { "id": "3a46b8c1-c2c6-3006-0000-000066d6dbdd", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Extract data attribute", "comments": "", "instanceIdentifier": "9e653a1f-9897-3124-8aa1-83321219f2fc" }, "destination": { "id": "dd4690f7-ae51-36bb-0000-0000325e1eed", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Produce station status records", "comments": "", "instanceIdentifier": "664aded4-922a-3da4-a584-7d5e43361201" }, "labelIndex": 0, "zIndex": 0, "selectedRelationships": [ "success" ], "backPressureObjectThreshold": 10000, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "138e8f4a-3dbd-3b9a-a259-c00bd60194c1", "instanceIdentifier": "796cfe4a-0f27-38df-a07f-03c8f172cc47", "name": "", "source": { "id": "c6929ff2-ea95-3d79-a483-3ffa75371ee4", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "SplitRecord", "comments": "", "instanceIdentifier": "78b9af30-7922-3dd6-91d4-f88d7f984ec6" }, "destination": { "id": "a1453322-1c3f-3150-a5be-0330c98144d9", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Extract station_uuid", "comments": "", "instanceIdentifier": "be5c2f57-09fd-3608-8c9c-9779e52c0ce8" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "splits" ], "backPressureObjectThreshold": 10, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "ROUND_ROBIN", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "d09052a3-4cb1-35c1-bed3-cbee709b53b3", "instanceIdentifier": "84221825-bfbc-3c5b-82bb-bfc3e21fcb48", "name": "", "source": { "id": "a1453322-1c3f-3150-a5be-0330c98144d9", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Extract station_uuid", "comments": "", "instanceIdentifier": "be5c2f57-09fd-3608-8c9c-9779e52c0ce8" }, "destination": { "id": "7133f561-971a-35b6-abff-9892e06b4bbe", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Get 30 days historic data", "comments": "", "instanceIdentifier": "fd7775f9-0f65-361a-bcb9-71668ae1a663" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "matched" ], "backPressureObjectThreshold": 10, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "572214c6-71ac-3533-964f-fd44e1ec17f2", "instanceIdentifier": "8171dd08-0195-1000-0000-000061d1719e", "name": "", "source": { "id": "405f050c-5005-3b96-ffff-ffffbc8a379e", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Extract data attribute", "comments": "", "instanceIdentifier": "3dd55349-9324-36ea-a6a0-d76280b79609" }, "destination": { "id": "90d02439-1516-3c8a-ffff-ffff8b2ffb20", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Produce station information records", "comments": "", "instanceIdentifier": "3a5f9f2c-bafa-3ae3-bc1b-2e8f1dbea59e" }, "labelIndex": 0, "zIndex": 0, "selectedRelationships": [ "success" ], "backPressureObjectThreshold": 10000, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "c75b4056-44a4-3f5e-93f8-e461b2550623", "instanceIdentifier": "373d1546-8ba2-3b9a-9c66-6b8f9b1917b8", "name": "", "source": { "id": "364e250b-f224-3e12-b4cf-c782c9a06bbe", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Get free bike status", "comments": "", "instanceIdentifier": "143b2ea1-f9cc-3140-afd1-426a846dd9d0" }, "destination": { "id": "056f4faa-a4dc-3024-ffff-ffffb73ef3a0", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Extract data attribute", "comments": "", "instanceIdentifier": "7b0b12f3-669e-32c4-b866-dea8a4a6f34a" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 10000, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "5448c0cf-2aa5-3ef8-8ecd-836ab3fdb853", "instanceIdentifier": "0eff3813-31ae-3e6e-9f91-8883c0b83ff4", "name": "", "source": { "id": "98c83f54-9f99-3d95-88b0-ae91140707c9", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Get station list", "comments": "", "instanceIdentifier": "098befce-068e-3d63-8586-1a7309e40e58" }, "destination": { "id": "c6929ff2-ea95-3d79-a483-3ffa75371ee4", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "SplitRecord", "comments": "", "instanceIdentifier": "78b9af30-7922-3dd6-91d4-f88d7f984ec6" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 1, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "9acf6812-3d3a-3f35-94e9-fa6eb209342f", "instanceIdentifier": "8170bfea-0195-1000-ffff-ffffdc31cffd", "name": "", "source": { "id": "42346af3-5793-3b8c-ffff-ffffc5c5139d", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Add station_uuid", "comments": "", "instanceIdentifier": "f9f95954-77fc-3f16-9b94-54ca68090b21" }, "destination": { "id": "762fb4f3-201b-3c92-ffff-ffff8e7e57fd", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "PublishKafka", "comments": "", "instanceIdentifier": "3d7eb954-e054-3324-aea1-5aa7a03dc120" }, "labelIndex": 0, "zIndex": 0, "selectedRelationships": [ "success" ], "backPressureObjectThreshold": 20, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "1ca5e75c-5900-3f28-94cb-25f1ade9a5a0", "instanceIdentifier": "ee452230-0ad0-3409-8b03-39acaf8ec15e", "name": "", "source": { "id": "0640256b-010a-3e69-b961-680e63ec85e2", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Extract station_uuid", "comments": "", "instanceIdentifier": "71fee077-9f08-3916-b189-f933b8f13619" }, "destination": { "id": "de71001e-de69-32c2-82b9-6f008665b252", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Get 30 minutes of historic data", "comments": "", "instanceIdentifier": "df138016-7764-3ad6-ac10-9f57e657461d" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "matched" ], "backPressureObjectThreshold": 10, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" } ], "labels": [ { "identifier": "f531aeda-2cad-3391-bc64-d10934a5f355", "instanceIdentifier": "f990fed8-9ffc-3b20-8795-efeaf1c55396", "position": { "x": 1309.0000007628792, "y": 241.99998844850802 }, "label": "Stream real-time data", "zIndex": 0, "width": 152.0, "height": 24.0, "style": { "font-size": "12px" }, "componentType": "LABEL", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "b4bf22b0-ff9e-3244-90bd-c6306a8e1fe5", "instanceIdentifier": "fdd434de-64f4-36e8-b1a8-63ad26549901", "position": { "x": 1701.0000007628792, "y": 201.99998844850802 }, "label": "Shared bikes data", "zIndex": 0, "width": 1152.0, "height": 24.0, "style": { "font-size": "12px" }, "componentType": "LABEL", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "637f34a6-0a08-34a9-bdcd-b3bfc4257a4c", "instanceIdentifier": "730910f3-60f8-3278-9098-5d864bca1ad3", "position": { "x": 2501.000000762879, "y": 241.99998844850802 }, "label": "Ingest bike status", "zIndex": 0, "width": 152.0, "height": 24.0, "style": { "font-size": "12px" }, "componentType": "LABEL", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "cb5fcdeb-c972-34cb-af5a-f6812416ed2d", "instanceIdentifier": "aafc5a3c-bbc6-3ef0-81e8-743f9dd99644", "position": { "x": 1701.0000007628792, "y": 241.99998844850802 }, "label": "Ingest station information", "zIndex": 0, "width": 152.0, "height": 24.0, "style": { "font-size": "12px" }, "componentType": "LABEL", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "f3bd6d3a-3dea-3909-a60e-aa3a34542e5c", "instanceIdentifier": "b8ec73a5-19b4-3d53-b152-7d99129c6342", "position": { "x": 557.000000762879, "y": 241.99998844850802 }, "label": "Ingest station records", "zIndex": 0, "width": 152.0, "height": 24.0, "style": { "font-size": "12px" }, "componentType": "LABEL", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "6dcd3b2a-e51d-329f-9aca-bffdc414bb51", "instanceIdentifier": "61a589ce-b17b-3090-ae4b-e9ce0dab0047", "position": { "x": 2101.000000762879, "y": 241.99998844850802 }, "label": "Ingest station status", "zIndex": 0, "width": 152.0, "height": 24.0, "style": { "font-size": "12px" }, "componentType": "LABEL", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "543a323b-3976-313c-9344-05c172ab5d02", "instanceIdentifier": "8e99d259-10ae-3f12-8b51-fdca63c2c201", "position": { "x": 933.000000762879, "y": 241.99998844850802 }, "label": "Ingest historic data", "zIndex": 0, "width": 152.0, "height": 24.0, "style": { "font-size": "12px" }, "componentType": "LABEL", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "f9ea8997-f618-3472-9745-93ca1036d176", "instanceIdentifier": "0393438f-a1a3-3b2a-8012-66c5050636cd", "position": { "x": 557.000000762879, "y": 201.99998844850802 }, "label": "Water levels data", "zIndex": 0, "width": 1104.0, "height": 24.0, "style": { "font-size": "12px" }, "componentType": "LABEL", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" } ], "funnels": [], "controllerServices": [ { "identifier": "906147e9-0ba3-3951-8764-15e14a9766d6", "instanceIdentifier": "31167665-331b-3abd-bbec-8026382f3e57", "name": "StandardRestrictedSSLContextService", "comments": "", "type": "org.apache.nifi.ssl.StandardRestrictedSSLContextService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-ssl-context-service-nar", "version": "2.6.0" }, "properties": { "Truststore Type": "PKCS12", "Truststore Password": "secret", "SSL Protocol": "TLSv1.3", "Keystore Password": "secret", "Keystore Type": "PKCS12", "Truststore Filename": "/stackable/keystore/truststore.p12", "Keystore Filename": "/stackable/keystore/keystore.p12" }, "propertyDescriptors": { "Truststore Type": { "name": "Truststore Type", "displayName": "Truststore Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Protocol": { "name": "SSL Protocol", "displayName": "TLS Protocol", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Keystore Type": { "name": "Keystore Type", "displayName": "Keystore Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Truststore Filename": { "name": "Truststore Filename", "displayName": "Truststore Filename", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "SINGLE", "resourceTypes": [ "FILE" ] } }, "Keystore Password": { "name": "Keystore Password", "displayName": "Keystore Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "key-password": { "name": "key-password", "displayName": "Key Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Truststore Password": { "name": "Truststore Password", "displayName": "Truststore Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Keystore Filename": { "name": "Keystore Filename", "displayName": "Keystore Filename", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "SINGLE", "resourceTypes": [ "FILE" ] } } }, "controllerServiceApis": [ { "type": "org.apache.nifi.ssl.SSLContextProvider", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } }, { "type": "org.apache.nifi.ssl.SSLContextService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } }, { "type": "org.apache.nifi.ssl.RestrictedSSLContextService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } } ], "scheduledState": "DISABLED", "bulletinLevel": "WARN", "componentType": "CONTROLLER_SERVICE", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "50e96367-7b84-3034-8dbf-6e02fa7c01df", "instanceIdentifier": "d795c667-0833-34e1-9d2b-4ae37d102704", "name": "JsonRecordSetWriter", "type": "org.apache.nifi.json.JsonRecordSetWriter", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-record-serialization-services-nar", "version": "2.6.0" }, "properties": { "schema-reference-reader": null, "schema-branch": null, "Allow Scientific Notation": "false", "compression-level": "1", "schema-cache": null, "Timestamp Format": null, "Date Format": null, "Pretty Print JSON": "false", "compression-format": "none", "Schema Write Strategy": "no-schema", "suppress-nulls": "never-suppress", "output-grouping": "output-array", "schema-name": "${schema.name}", "schema-registry": null, "Time Format": null, "schema-access-strategy": "inherit-record-schema", "schema-version": null, "schema-text": "${avro.schema}", "Schema Reference Writer": null }, "propertyDescriptors": { "schema-reference-reader": { "name": "schema-reference-reader", "displayName": "Schema Reference Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "schema-branch": { "name": "schema-branch", "displayName": "Schema Branch", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Allow Scientific Notation": { "name": "Allow Scientific Notation", "displayName": "Allow Scientific Notation", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "compression-level": { "name": "compression-level", "displayName": "Compression Level", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-cache": { "name": "schema-cache", "displayName": "Schema Cache", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Timestamp Format": { "name": "Timestamp Format", "displayName": "Timestamp Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Date Format": { "name": "Date Format", "displayName": "Date Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Pretty Print JSON": { "name": "Pretty Print JSON", "displayName": "Pretty Print JSON", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "compression-format": { "name": "compression-format", "displayName": "Compression Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Schema Write Strategy": { "name": "Schema Write Strategy", "displayName": "Schema Write Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "suppress-nulls": { "name": "suppress-nulls", "displayName": "Suppress Null Values", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "output-grouping": { "name": "output-grouping", "displayName": "Output Grouping", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-name": { "name": "schema-name", "displayName": "Schema Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-registry": { "name": "schema-registry", "displayName": "Schema Registry", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Time Format": { "name": "Time Format", "displayName": "Time Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-access-strategy": { "name": "schema-access-strategy", "displayName": "Schema Access Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-version": { "name": "schema-version", "displayName": "Schema Version", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-text": { "name": "schema-text", "displayName": "Schema Text", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Schema Reference Writer": { "name": "Schema Reference Writer", "displayName": "Schema Reference Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false } }, "controllerServiceApis": [ { "type": "org.apache.nifi.serialization.RecordSetWriterFactory", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } } ], "scheduledState": "DISABLED", "bulletinLevel": "WARN", "componentType": "CONTROLLER_SERVICE", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "a7cc755f-2210-3763-889e-24c3e0b0684d", "instanceIdentifier": "8167b5da-0195-1000-ffff-ffff926c765c", "name": "Kafka3ConnectionService", "comments": "", "type": "org.apache.nifi.kafka.service.Kafka3ConnectionService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-3-service-nar", "version": "2.6.0" }, "properties": { "security.protocol": "SSL", "SSL Context Service": "906147e9-0ba3-3951-8764-15e14a9766d6", "ack.wait.time": "5 sec", "sasl.username": null, "max.poll.records": "10000", "sasl.mechanism": "GSSAPI", "isolation.level": "read_committed", "max.block.ms": "5 sec", "bootstrap.servers": "kafka-broker-default-headless.default.svc.cluster.local:9093", "kerberos-user-service": null, "sasl.kerberos.service.name": null, "default.api.timeout.ms": "60 sec" }, "propertyDescriptors": { "ack.wait.time": { "name": "ack.wait.time", "displayName": "Acknowledgment Wait Time", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "sasl.username": { "name": "sasl.username", "displayName": "SASL Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.poll.records": { "name": "max.poll.records", "displayName": "Max Poll Records", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "bootstrap.servers": { "name": "bootstrap.servers", "displayName": "Bootstrap Servers", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "sasl.kerberos.service.name": { "name": "sasl.kerberos.service.name", "displayName": "Kerberos Service Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "security.protocol": { "name": "security.protocol", "displayName": "Security Protocol", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "sasl.mechanism": { "name": "sasl.mechanism", "displayName": "SASL Mechanism", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "isolation.level": { "name": "isolation.level", "displayName": "Transaction Isolation Level", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.block.ms": { "name": "max.block.ms", "displayName": "Max Metadata Wait Time", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "kerberos-user-service": { "name": "kerberos-user-service", "displayName": "Kerberos User Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "sasl.password": { "name": "sasl.password", "displayName": "SASL Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "default.api.timeout.ms": { "name": "default.api.timeout.ms", "displayName": "Client Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "controllerServiceApis": [ { "type": "org.apache.nifi.kafka.service.api.KafkaConnectionService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-service-api-nar", "version": "2.6.0" } } ], "scheduledState": "DISABLED", "bulletinLevel": "WARN", "componentType": "CONTROLLER_SERVICE", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "c127588a-e95f-382f-afb1-33f9b279c38d", "instanceIdentifier": "715ff6c2-fd41-366f-9922-e0365c063f33", "name": "JsonTreeReader", "type": "org.apache.nifi.json.JsonTreeReader", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-record-serialization-services-nar", "version": "2.6.0" }, "properties": { "schema-reference-reader": null, "schema-branch": null, "Max String Length": "20 MB", "schema-application-strategy": "SELECTED_PART", "Timestamp Format": null, "schema-inference-cache": null, "Date Format": null, "schema-name": "${schema.name}", "starting-field-strategy": "ROOT_NODE", "schema-registry": null, "starting-field-name": null, "Time Format": null, "schema-access-strategy": "infer-schema", "schema-version": null, "schema-text": "${avro.schema}", "Allow Comments": "false" }, "propertyDescriptors": { "schema-reference-reader": { "name": "schema-reference-reader", "displayName": "Schema Reference Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "schema-branch": { "name": "schema-branch", "displayName": "Schema Branch", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Max String Length": { "name": "Max String Length", "displayName": "Max String Length", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-application-strategy": { "name": "schema-application-strategy", "displayName": "Schema Application Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Timestamp Format": { "name": "Timestamp Format", "displayName": "Timestamp Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-inference-cache": { "name": "schema-inference-cache", "displayName": "Schema Inference Cache", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Date Format": { "name": "Date Format", "displayName": "Date Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-name": { "name": "schema-name", "displayName": "Schema Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "starting-field-strategy": { "name": "starting-field-strategy", "displayName": "Starting Field Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-registry": { "name": "schema-registry", "displayName": "Schema Registry", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "starting-field-name": { "name": "starting-field-name", "displayName": "Starting Field Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Time Format": { "name": "Time Format", "displayName": "Time Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-access-strategy": { "name": "schema-access-strategy", "displayName": "Schema Access Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-version": { "name": "schema-version", "displayName": "Schema Version", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-text": { "name": "schema-text", "displayName": "Schema Text", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Allow Comments": { "name": "Allow Comments", "displayName": "Allow Comments", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "controllerServiceApis": [ { "type": "org.apache.nifi.serialization.RecordReaderFactory", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } } ], "scheduledState": "DISABLED", "bulletinLevel": "WARN", "componentType": "CONTROLLER_SERVICE", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" } ], "defaultFlowFileExpiration": "0 sec", "defaultBackPressureObjectThreshold": 10000, "defaultBackPressureDataSizeThreshold": "1 GB", "scheduledState": "ENABLED", "executionEngine": "INHERITED", "maxConcurrentTasks": 1, "statelessFlowTimeout": "1 min", "flowFileConcurrency": "UNBOUNDED", "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE", "componentType": "PROCESS_GROUP" }, "externalControllerServices": {}, "parameterContexts": {}, "flowEncodingVersion": "1.0", "parameterProviders": {}, "latest": false } + { "flowContents": { "identifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "instanceIdentifier": "8166cd44-0195-1000-ffff-ffffe3e76943", "name": "LakehouseKafkaIngest_raw", "comments": "", "position": { "x": -40.0, "y": -416.0 }, "processGroups": [], "remoteProcessGroups": [], "processors": [ { "identifier": "5e50831e-a2fd-34c9-af0a-924b16280705", "instanceIdentifier": "54f2ba99-3a49-3160-b8b9-45b8c0a6ac2a", "name": "Get station information", "comments": "", "position": { "x": 1701.0000007628792, "y": 273.999988448508 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://api.mobidata-bw.de/sharing/gbfs/v2/deer/station_information", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "false", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "10 m", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "PRIMARY", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "ba73d7c1-102f-3888-a573-5e6d2ec9e1f0", "instanceIdentifier": "816f11f1-0195-1000-ffff-ffffd698d6c2", "name": "Add station_uuid", "comments": "", "position": { "x": 936.0, "y": 1216.0 }, "type": "org.apache.nifi.processors.jolt.JoltTransformJSON", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-jolt-nar", "version": "2.6.0" }, "properties": { "Jolt Transform": "jolt-transform-chain", "Max String Length": "20 MB", "Transform Cache Size": "1", "Custom Transformation Class Name": null, "Custom Module Directory": null, "Pretty Print": "false", "Jolt Specification": "[\n {\n \"operation\": \"default\",\n \"spec\": {\n \"*\": {\n \"station_uuid\": \"${station_uuid}\"\n }\n }\n }\n]\n" }, "propertyDescriptors": { "Jolt Transform": { "name": "Jolt Transform", "displayName": "Jolt Transform", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Max String Length": { "name": "Max String Length", "displayName": "Max String Length", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transform Cache Size": { "name": "Transform Cache Size", "displayName": "Transform Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Transformation Class Name": { "name": "Custom Transformation Class Name", "displayName": "Custom Transformation Class Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Module Directory": { "name": "Custom Module Directory", "displayName": "Custom Module Directory", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "MULTIPLE", "resourceTypes": [ "DIRECTORY", "FILE" ] } }, "Pretty Print": { "name": "Pretty Print", "displayName": "Pretty Print", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Jolt Specification": { "name": "Jolt Specification", "displayName": "Jolt Specification", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "SINGLE", "resourceTypes": [ "TEXT", "FILE" ] } } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "577f603d-ee9c-3202-8139-d35d94c9c311", "instanceIdentifier": "331d7feb-96a3-3998-9a1e-5b23cca85c5f", "name": "SplitRecord", "comments": "", "position": { "x": 1309.0000007628792, "y": 537.999988448508 }, "type": "org.apache.nifi.processors.standard.SplitRecord", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Record Writer": "50e96367-7b84-3034-8dbf-6e02fa7c01df", "Record Reader": "c127588a-e95f-382f-afb1-33f9b279c38d", "Records Per Split": "1" }, "propertyDescriptors": { "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Records Per Split": { "name": "Records Per Split", "displayName": "Records Per Split", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "original", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "fc594301-ecc4-353b-8330-7aecb14e6422", "instanceIdentifier": "f049f0b7-46f9-332b-bce3-925167b05f37", "name": "Get station list", "comments": "", "position": { "x": 1309.0000007628792, "y": 273.999988448508 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations.json", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "false", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "15 m", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "PRIMARY", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "3a46b8c1-c2c6-3006-0000-000066d6dbdd", "instanceIdentifier": "9e653a1f-9897-3124-8aa1-83321219f2fc", "name": "Extract data attribute", "comments": "", "position": { "x": 2104.0, "y": 536.0 }, "type": "org.apache.nifi.processors.jolt.JoltTransformJSON", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-jolt-nar", "version": "2.6.0" }, "properties": { "Jolt Transform": "jolt-transform-chain", "Max String Length": "20 MB", "Transform Cache Size": "1", "Custom Transformation Class Name": null, "Custom Module Directory": null, "Pretty Print": "false", "Jolt Specification": "[\n {\n \"operation\": \"shift\",\n \"spec\": {\n \"data\": {\n \"*\": \"\"\n }\n }\n }\n]\n" }, "propertyDescriptors": { "Jolt Transform": { "name": "Jolt Transform", "displayName": "Jolt Transform", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Max String Length": { "name": "Max String Length", "displayName": "Max String Length", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transform Cache Size": { "name": "Transform Cache Size", "displayName": "Transform Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Transformation Class Name": { "name": "Custom Transformation Class Name", "displayName": "Custom Transformation Class Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Module Directory": { "name": "Custom Module Directory", "displayName": "Custom Module Directory", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "MULTIPLE", "resourceTypes": [ "DIRECTORY", "FILE" ] } }, "Pretty Print": { "name": "Pretty Print", "displayName": "Pretty Print", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Jolt Specification": { "name": "Jolt Specification", "displayName": "Jolt Specification", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "SINGLE", "resourceTypes": [ "TEXT", "FILE" ] } } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "dd4690f7-ae51-36bb-0000-0000325e1eed", "instanceIdentifier": "664aded4-922a-3da4-a584-7d5e43361201", "name": "Produce station status records", "comments": "", "position": { "x": 2104.0, "y": 752.0 }, "type": "org.apache.nifi.kafka.processors.PublishKafka", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-nar", "version": "2.6.0" }, "properties": { "compression.type": "none", "acks": "all", "Message Demarcator": null, "Kafka Key Attribute Encoding": "utf-8", "Record Reader": "c127588a-e95f-382f-afb1-33f9b279c38d", "Record Metadata Strategy": "FROM_PROPERTIES", "Transactional ID Prefix": null, "Header Encoding": "UTF-8", "max.request.size": "1 MB", "Topic Name": "shared_bikes_station_status", "Kafka Key": null, "partition": null, "Kafka Connection Service": "a7cc755f-2210-3763-889e-24c3e0b0684d", "Publish Strategy": "USE_VALUE", "Record Key Writer": null, "Failure Strategy": "Route to Failure", "partitioner.class": "org.apache.kafka.clients.producer.internals.DefaultPartitioner", "Record Writer": "50e96367-7b84-3034-8dbf-6e02fa7c01df", "Transactions Enabled": "false", "Message Key Field": null, "FlowFile Attribute Header Pattern": null }, "propertyDescriptors": { "compression.type": { "name": "compression.type", "displayName": "Compression Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "acks": { "name": "acks", "displayName": "Delivery Guarantee", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Demarcator": { "name": "Message Demarcator", "displayName": "Message Demarcator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key Attribute Encoding": { "name": "Kafka Key Attribute Encoding", "displayName": "Kafka Key Attribute Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Metadata Strategy": { "name": "Record Metadata Strategy", "displayName": "Record Metadata Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transactional ID Prefix": { "name": "Transactional ID Prefix", "displayName": "Transactional ID Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Header Encoding": { "name": "Header Encoding", "displayName": "Header Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.request.size": { "name": "max.request.size", "displayName": "Max Request Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Topic Name": { "name": "Topic Name", "displayName": "Topic Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key": { "name": "Kafka Key", "displayName": "Kafka Key", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partition": { "name": "partition", "displayName": "Partition", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Connection Service": { "name": "Kafka Connection Service", "displayName": "Kafka Connection Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Publish Strategy": { "name": "Publish Strategy", "displayName": "Publish Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Key Writer": { "name": "Record Key Writer", "displayName": "Record Key Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Failure Strategy": { "name": "Failure Strategy", "displayName": "Failure Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partitioner.class": { "name": "partitioner.class", "displayName": "Partitioner Class", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Transactions Enabled": { "name": "Transactions Enabled", "displayName": "Transactions Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Key Field": { "name": "Message Key Field", "displayName": "Message Key Field", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "FlowFile Attribute Header Pattern": { "name": "FlowFile Attribute Header Pattern", "displayName": "FlowFile Attribute Header Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "success", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "90d02439-1516-3c8a-ffff-ffff8b2ffb20", "instanceIdentifier": "3a5f9f2c-bafa-3ae3-bc1b-2e8f1dbea59e", "name": "Produce station information records", "comments": "", "position": { "x": 1704.0, "y": 752.0 }, "type": "org.apache.nifi.kafka.processors.PublishKafka", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-nar", "version": "2.6.0" }, "properties": { "compression.type": "none", "acks": "all", "Message Demarcator": null, "Kafka Key Attribute Encoding": "utf-8", "Record Reader": "c127588a-e95f-382f-afb1-33f9b279c38d", "Record Metadata Strategy": "FROM_PROPERTIES", "Transactional ID Prefix": null, "Header Encoding": "UTF-8", "max.request.size": "1 MB", "Topic Name": "shared_bikes_station_information", "Kafka Key": null, "partition": null, "Kafka Connection Service": "a7cc755f-2210-3763-889e-24c3e0b0684d", "Publish Strategy": "USE_VALUE", "Record Key Writer": null, "Failure Strategy": "Route to Failure", "partitioner.class": "org.apache.kafka.clients.producer.internals.DefaultPartitioner", "Record Writer": "50e96367-7b84-3034-8dbf-6e02fa7c01df", "Transactions Enabled": "false", "Message Key Field": null, "FlowFile Attribute Header Pattern": null }, "propertyDescriptors": { "compression.type": { "name": "compression.type", "displayName": "Compression Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "acks": { "name": "acks", "displayName": "Delivery Guarantee", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Demarcator": { "name": "Message Demarcator", "displayName": "Message Demarcator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key Attribute Encoding": { "name": "Kafka Key Attribute Encoding", "displayName": "Kafka Key Attribute Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Metadata Strategy": { "name": "Record Metadata Strategy", "displayName": "Record Metadata Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transactional ID Prefix": { "name": "Transactional ID Prefix", "displayName": "Transactional ID Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Header Encoding": { "name": "Header Encoding", "displayName": "Header Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.request.size": { "name": "max.request.size", "displayName": "Max Request Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Topic Name": { "name": "Topic Name", "displayName": "Topic Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key": { "name": "Kafka Key", "displayName": "Kafka Key", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partition": { "name": "partition", "displayName": "Partition", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Connection Service": { "name": "Kafka Connection Service", "displayName": "Kafka Connection Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Publish Strategy": { "name": "Publish Strategy", "displayName": "Publish Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Key Writer": { "name": "Record Key Writer", "displayName": "Record Key Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Failure Strategy": { "name": "Failure Strategy", "displayName": "Failure Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partitioner.class": { "name": "partitioner.class", "displayName": "Partitioner Class", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Transactions Enabled": { "name": "Transactions Enabled", "displayName": "Transactions Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Key Field": { "name": "Message Key Field", "displayName": "Message Key Field", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "FlowFile Attribute Header Pattern": { "name": "FlowFile Attribute Header Pattern", "displayName": "FlowFile Attribute Header Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "success", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "056f4faa-a4dc-3024-ffff-ffffb73ef3a0", "instanceIdentifier": "7b0b12f3-669e-32c4-b866-dea8a4a6f34a", "name": "Extract data attribute", "comments": "", "position": { "x": 2496.0, "y": 536.0 }, "type": "org.apache.nifi.processors.jolt.JoltTransformJSON", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-jolt-nar", "version": "2.6.0" }, "properties": { "Jolt Transform": "jolt-transform-chain", "Max String Length": "20 MB", "Transform Cache Size": "1", "Custom Transformation Class Name": null, "Custom Module Directory": null, "Pretty Print": "false", "Jolt Specification": "[\n {\n \"operation\": \"shift\",\n \"spec\": {\n \"data\": {\n \"*\": \"\"\n }\n }\n }\n]\n" }, "propertyDescriptors": { "Jolt Transform": { "name": "Jolt Transform", "displayName": "Jolt Transform", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Max String Length": { "name": "Max String Length", "displayName": "Max String Length", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transform Cache Size": { "name": "Transform Cache Size", "displayName": "Transform Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Transformation Class Name": { "name": "Custom Transformation Class Name", "displayName": "Custom Transformation Class Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Module Directory": { "name": "Custom Module Directory", "displayName": "Custom Module Directory", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "MULTIPLE", "resourceTypes": [ "DIRECTORY", "FILE" ] } }, "Pretty Print": { "name": "Pretty Print", "displayName": "Pretty Print", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Jolt Specification": { "name": "Jolt Specification", "displayName": "Jolt Specification", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "SINGLE", "resourceTypes": [ "TEXT", "FILE" ] } } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "42346af3-5793-3b8c-ffff-ffffc5c5139d", "instanceIdentifier": "f9f95954-77fc-3f16-9b94-54ca68090b21", "name": "Add station_uuid", "comments": "", "position": { "x": 1312.0, "y": 1216.0 }, "type": "org.apache.nifi.processors.jolt.JoltTransformJSON", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-jolt-nar", "version": "2.6.0" }, "properties": { "Jolt Transform": "jolt-transform-chain", "Max String Length": "20 MB", "Transform Cache Size": "1", "Custom Transformation Class Name": null, "Custom Module Directory": null, "Pretty Print": "false", "Jolt Specification": "[\n {\n \"operation\": \"default\",\n \"spec\": {\n \"*\": {\n \"station_uuid\": \"${station_uuid}\"\n }\n }\n }\n]\n" }, "propertyDescriptors": { "Jolt Transform": { "name": "Jolt Transform", "displayName": "Jolt Transform", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Max String Length": { "name": "Max String Length", "displayName": "Max String Length", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transform Cache Size": { "name": "Transform Cache Size", "displayName": "Transform Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Transformation Class Name": { "name": "Custom Transformation Class Name", "displayName": "Custom Transformation Class Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Module Directory": { "name": "Custom Module Directory", "displayName": "Custom Module Directory", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "MULTIPLE", "resourceTypes": [ "DIRECTORY", "FILE" ] } }, "Pretty Print": { "name": "Pretty Print", "displayName": "Pretty Print", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Jolt Specification": { "name": "Jolt Specification", "displayName": "Jolt Specification", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "SINGLE", "resourceTypes": [ "TEXT", "FILE" ] } } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "1a0aea35-946c-35de-8811-549faeb003ee", "instanceIdentifier": "81675904-0195-1000-ffff-ffffbbbb9cfe", "name": "Produce station records", "comments": "", "position": { "x": 560.0, "y": 536.0 }, "type": "org.apache.nifi.kafka.processors.PublishKafka", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-nar", "version": "2.6.0" }, "properties": { "compression.type": "none", "acks": "all", "Message Demarcator": null, "Kafka Key Attribute Encoding": "utf-8", "Record Reader": "c127588a-e95f-382f-afb1-33f9b279c38d", "Record Metadata Strategy": "FROM_PROPERTIES", "Transactional ID Prefix": null, "Header Encoding": "UTF-8", "max.request.size": "1 MB", "Topic Name": "water_levels_stations", "Kafka Key": null, "partition": null, "Kafka Connection Service": "a7cc755f-2210-3763-889e-24c3e0b0684d", "Publish Strategy": "USE_VALUE", "Record Key Writer": null, "Failure Strategy": "Route to Failure", "partitioner.class": "org.apache.kafka.clients.producer.internals.DefaultPartitioner", "Record Writer": "50e96367-7b84-3034-8dbf-6e02fa7c01df", "Transactions Enabled": "false", "Message Key Field": null, "FlowFile Attribute Header Pattern": null }, "propertyDescriptors": { "compression.type": { "name": "compression.type", "displayName": "Compression Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "acks": { "name": "acks", "displayName": "Delivery Guarantee", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Demarcator": { "name": "Message Demarcator", "displayName": "Message Demarcator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key Attribute Encoding": { "name": "Kafka Key Attribute Encoding", "displayName": "Kafka Key Attribute Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Metadata Strategy": { "name": "Record Metadata Strategy", "displayName": "Record Metadata Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transactional ID Prefix": { "name": "Transactional ID Prefix", "displayName": "Transactional ID Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Header Encoding": { "name": "Header Encoding", "displayName": "Header Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.request.size": { "name": "max.request.size", "displayName": "Max Request Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Topic Name": { "name": "Topic Name", "displayName": "Topic Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key": { "name": "Kafka Key", "displayName": "Kafka Key", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partition": { "name": "partition", "displayName": "Partition", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Connection Service": { "name": "Kafka Connection Service", "displayName": "Kafka Connection Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Publish Strategy": { "name": "Publish Strategy", "displayName": "Publish Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Key Writer": { "name": "Record Key Writer", "displayName": "Record Key Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Failure Strategy": { "name": "Failure Strategy", "displayName": "Failure Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partitioner.class": { "name": "partitioner.class", "displayName": "Partitioner Class", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Transactions Enabled": { "name": "Transactions Enabled", "displayName": "Transactions Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Key Field": { "name": "Message Key Field", "displayName": "Message Key Field", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "FlowFile Attribute Header Pattern": { "name": "FlowFile Attribute Header Pattern", "displayName": "FlowFile Attribute Header Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "success", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "364e250b-f224-3e12-b4cf-c782c9a06bbe", "instanceIdentifier": "143b2ea1-f9cc-3140-afd1-426a846dd9d0", "name": "Get free bike status", "comments": "", "position": { "x": 2501.000000762879, "y": 273.999988448508 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://api.mobidata-bw.de/sharing/gbfs/v2/deer/free_bike_status", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "false", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "1 m", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "PRIMARY", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "26a592be-7444-337c-b8f9-3a29013cb760", "instanceIdentifier": "0ea082f4-3f0d-38db-bd4b-51feb5428cc6", "name": "Get station status", "comments": "", "position": { "x": 2101.000000762879, "y": 273.999988448508 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://api.mobidata-bw.de/sharing/gbfs/v2/deer/station_status", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "false", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "1 m", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "PRIMARY", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "d9bacb7c-10df-3044-ffff-ffffe7a604af", "instanceIdentifier": "8c7c157b-6f95-399c-83f7-54a79ea11078", "name": "Produce bike status records", "comments": "", "position": { "x": 2504.0, "y": 752.0 }, "type": "org.apache.nifi.kafka.processors.PublishKafka", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-nar", "version": "2.6.0" }, "properties": { "compression.type": "none", "acks": "all", "Message Demarcator": null, "Kafka Key Attribute Encoding": "utf-8", "Record Reader": "c127588a-e95f-382f-afb1-33f9b279c38d", "Record Metadata Strategy": "FROM_PROPERTIES", "Transactional ID Prefix": null, "Header Encoding": "UTF-8", "max.request.size": "1 MB", "Topic Name": "shared_bikes_bike_status", "Kafka Key": null, "partition": null, "Kafka Connection Service": "a7cc755f-2210-3763-889e-24c3e0b0684d", "Publish Strategy": "USE_VALUE", "Record Key Writer": null, "Failure Strategy": "Route to Failure", "partitioner.class": "org.apache.kafka.clients.producer.internals.DefaultPartitioner", "Record Writer": "50e96367-7b84-3034-8dbf-6e02fa7c01df", "Transactions Enabled": "false", "Message Key Field": null, "FlowFile Attribute Header Pattern": null }, "propertyDescriptors": { "compression.type": { "name": "compression.type", "displayName": "Compression Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "acks": { "name": "acks", "displayName": "Delivery Guarantee", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Demarcator": { "name": "Message Demarcator", "displayName": "Message Demarcator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key Attribute Encoding": { "name": "Kafka Key Attribute Encoding", "displayName": "Kafka Key Attribute Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Metadata Strategy": { "name": "Record Metadata Strategy", "displayName": "Record Metadata Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transactional ID Prefix": { "name": "Transactional ID Prefix", "displayName": "Transactional ID Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Header Encoding": { "name": "Header Encoding", "displayName": "Header Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.request.size": { "name": "max.request.size", "displayName": "Max Request Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Topic Name": { "name": "Topic Name", "displayName": "Topic Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key": { "name": "Kafka Key", "displayName": "Kafka Key", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partition": { "name": "partition", "displayName": "Partition", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Connection Service": { "name": "Kafka Connection Service", "displayName": "Kafka Connection Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Publish Strategy": { "name": "Publish Strategy", "displayName": "Publish Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Key Writer": { "name": "Record Key Writer", "displayName": "Record Key Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Failure Strategy": { "name": "Failure Strategy", "displayName": "Failure Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partitioner.class": { "name": "partitioner.class", "displayName": "Partitioner Class", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Transactions Enabled": { "name": "Transactions Enabled", "displayName": "Transactions Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Key Field": { "name": "Message Key Field", "displayName": "Message Key Field", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "FlowFile Attribute Header Pattern": { "name": "FlowFile Attribute Header Pattern", "displayName": "FlowFile Attribute Header Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "success", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "00bcf948-7bef-35b6-8a6d-e641194b8189", "instanceIdentifier": "d92196ba-38f9-3b77-b9a4-4e0dfe564d3f", "name": "Get station list", "comments": "", "position": { "x": 557.000000762879, "y": 273.999988448508 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations.json", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "false", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "10 m", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "PRIMARY", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "98c83f54-9f99-3d95-88b0-ae91140707c9", "instanceIdentifier": "098befce-068e-3d63-8586-1a7309e40e58", "name": "Get station list", "comments": "", "position": { "x": 933.000000762879, "y": 273.999988448508 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations.json", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "false", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "365000 days", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "PRIMARY", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "405f050c-5005-3b96-ffff-ffffbc8a379e", "instanceIdentifier": "3dd55349-9324-36ea-a6a0-d76280b79609", "name": "Extract data attribute", "comments": "", "position": { "x": 1704.0, "y": 544.0 }, "type": "org.apache.nifi.processors.jolt.JoltTransformJSON", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-jolt-nar", "version": "2.6.0" }, "properties": { "Jolt Transform": "jolt-transform-chain", "Max String Length": "20 MB", "Transform Cache Size": "1", "Custom Transformation Class Name": null, "Custom Module Directory": null, "Pretty Print": "false", "Jolt Specification": "[\n {\n \"operation\": \"shift\",\n \"spec\": {\n \"data\": {\n \"*\": \"\"\n }\n }\n }\n]\n" }, "propertyDescriptors": { "Jolt Transform": { "name": "Jolt Transform", "displayName": "Jolt Transform", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Max String Length": { "name": "Max String Length", "displayName": "Max String Length", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transform Cache Size": { "name": "Transform Cache Size", "displayName": "Transform Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Transformation Class Name": { "name": "Custom Transformation Class Name", "displayName": "Custom Transformation Class Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Custom Module Directory": { "name": "Custom Module Directory", "displayName": "Custom Module Directory", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "MULTIPLE", "resourceTypes": [ "DIRECTORY", "FILE" ] } }, "Pretty Print": { "name": "Pretty Print", "displayName": "Pretty Print", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Jolt Specification": { "name": "Jolt Specification", "displayName": "Jolt Specification", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "SINGLE", "resourceTypes": [ "TEXT", "FILE" ] } } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "a1453322-1c3f-3150-a5be-0330c98144d9", "instanceIdentifier": "be5c2f57-09fd-3608-8c9c-9779e52c0ce8", "name": "Extract station_uuid", "comments": "", "position": { "x": 933.000000762879, "y": 753.999988448508 }, "type": "org.apache.nifi.processors.standard.EvaluateJsonPath", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Destination": "flowfile-attribute", "Max String Length": "20 MB", "Return Type": "auto-detect", "Null Value Representation": "empty string", "station_uuid": "$.uuid", "Path Not Found Behavior": "ignore" }, "propertyDescriptors": { "Destination": { "name": "Destination", "displayName": "Destination", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Max String Length": { "name": "Max String Length", "displayName": "Max String Length", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Return Type": { "name": "Return Type", "displayName": "Return Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Null Value Representation": { "name": "Null Value Representation", "displayName": "Null Value Representation", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "station_uuid": { "name": "station_uuid", "displayName": "station_uuid", "identifiesControllerService": false, "sensitive": false, "dynamic": true }, "Path Not Found Behavior": { "name": "Path Not Found Behavior", "displayName": "Path Not Found Behavior", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "failure", "unmatched" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "de71001e-de69-32c2-82b9-6f008665b252", "instanceIdentifier": "df138016-7764-3ad6-ac10-9f57e657461d", "name": "Get 30 minutes of historic data", "comments": "", "position": { "x": 1309.0000007628792, "y": 993.999988448508 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations/${station_uuid}/W/measurements.json?start=PT30M", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "false", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "1 s", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "7133f561-971a-35b6-abff-9892e06b4bbe", "instanceIdentifier": "fd7775f9-0f65-361a-bcb9-71668ae1a663", "name": "Get 30 days historic data", "comments": "", "position": { "x": 933.000000762879, "y": 993.999988448508 }, "type": "org.apache.nifi.processors.standard.InvokeHTTP", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Request Content-Encoding": "DISABLED", "proxy-configuration-service": null, "Request Multipart Form-Data Filename Enabled": "true", "Request Chunked Transfer-Encoding Enabled": "false", "Response Header Request Attributes Prefix": null, "HTTP/2 Disabled": "False", "Connection Timeout": "5 secs", "Response Cookie Strategy": "DISABLED", "Socket Read Timeout": "15 secs", "Socket Idle Connections": "5", "Request Body Enabled": "true", "HTTP URL": "https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations/${station_uuid}/W/measurements.json?start=P30D", "Request OAuth2 Access Token Provider": null, "Socket Idle Timeout": "5 mins", "Response Redirects Enabled": "True", "Socket Write Timeout": "15 secs", "Request Header Attributes Pattern": null, "Response FlowFile Naming Strategy": "RANDOM", "Response Cache Enabled": "false", "Request Date Header Enabled": "True", "Request Failure Penalization Enabled": "false", "Response Body Attribute Size": "256", "SSL Context Service": null, "Response Generation Required": "false", "Request User-Agent": null, "Response Header Request Attributes Enabled": "false", "HTTP Method": "GET", "Request Username": null, "Request Content-Type": "${mime.type}", "Response Body Attribute Name": null, "Request Digest Authentication Enabled": "false", "Request Multipart Form-Data Name": null, "Response Cache Size": "10MB", "Response Body Ignored": "false" }, "propertyDescriptors": { "Request Content-Encoding": { "name": "Request Content-Encoding", "displayName": "Request Content-Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "proxy-configuration-service": { "name": "proxy-configuration-service", "displayName": "Proxy Configuration Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Filename Enabled": { "name": "Request Multipart Form-Data Filename Enabled", "displayName": "Request Multipart Form-Data Filename Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Chunked Transfer-Encoding Enabled": { "name": "Request Chunked Transfer-Encoding Enabled", "displayName": "Request Chunked Transfer-Encoding Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Prefix": { "name": "Response Header Request Attributes Prefix", "displayName": "Response Header Request Attributes Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP/2 Disabled": { "name": "HTTP/2 Disabled", "displayName": "HTTP/2 Disabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Connection Timeout": { "name": "Connection Timeout", "displayName": "Connection Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cookie Strategy": { "name": "Response Cookie Strategy", "displayName": "Response Cookie Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Password": { "name": "Request Password", "displayName": "Request Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Socket Read Timeout": { "name": "Socket Read Timeout", "displayName": "Socket Read Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Idle Connections": { "name": "Socket Idle Connections", "displayName": "Socket Idle Connections", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Body Enabled": { "name": "Request Body Enabled", "displayName": "Request Body Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP URL": { "name": "HTTP URL", "displayName": "HTTP URL", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request OAuth2 Access Token Provider": { "name": "Request OAuth2 Access Token Provider", "displayName": "Request OAuth2 Access Token Provider", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Socket Idle Timeout": { "name": "Socket Idle Timeout", "displayName": "Socket Idle Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Redirects Enabled": { "name": "Response Redirects Enabled", "displayName": "Response Redirects Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Socket Write Timeout": { "name": "Socket Write Timeout", "displayName": "Socket Write Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Header Attributes Pattern": { "name": "Request Header Attributes Pattern", "displayName": "Request Header Attributes Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response FlowFile Naming Strategy": { "name": "Response FlowFile Naming Strategy", "displayName": "Response FlowFile Naming Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Enabled": { "name": "Response Cache Enabled", "displayName": "Response Cache Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Date Header Enabled": { "name": "Request Date Header Enabled", "displayName": "Request Date Header Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Failure Penalization Enabled": { "name": "Request Failure Penalization Enabled", "displayName": "Request Failure Penalization Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Size": { "name": "Response Body Attribute Size", "displayName": "Response Body Attribute Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Response Generation Required": { "name": "Response Generation Required", "displayName": "Response Generation Required", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request User-Agent": { "name": "Request User-Agent", "displayName": "Request User-Agent", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Header Request Attributes Enabled": { "name": "Response Header Request Attributes Enabled", "displayName": "Response Header Request Attributes Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "HTTP Method": { "name": "HTTP Method", "displayName": "HTTP Method", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Username": { "name": "Request Username", "displayName": "Request Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Content-Type": { "name": "Request Content-Type", "displayName": "Request Content-Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Attribute Name": { "name": "Response Body Attribute Name", "displayName": "Response Body Attribute Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Digest Authentication Enabled": { "name": "Request Digest Authentication Enabled", "displayName": "Request Digest Authentication Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Request Multipart Form-Data Name": { "name": "Request Multipart Form-Data Name", "displayName": "Request Multipart Form-Data Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Cache Size": { "name": "Response Cache Size", "displayName": "Response Cache Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Response Body Ignored": { "name": "Response Body Ignored", "displayName": "Response Body Ignored", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "No Retry", "Retry", "Original", "Failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "0640256b-010a-3e69-b961-680e63ec85e2", "instanceIdentifier": "71fee077-9f08-3916-b189-f933b8f13619", "name": "Extract station_uuid", "comments": "", "position": { "x": 1309.0000007628792, "y": 753.999988448508 }, "type": "org.apache.nifi.processors.standard.EvaluateJsonPath", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Destination": "flowfile-attribute", "Max String Length": "20 MB", "Return Type": "auto-detect", "Null Value Representation": "empty string", "station_uuid": "$.uuid", "Path Not Found Behavior": "ignore" }, "propertyDescriptors": { "Destination": { "name": "Destination", "displayName": "Destination", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Max String Length": { "name": "Max String Length", "displayName": "Max String Length", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Return Type": { "name": "Return Type", "displayName": "Return Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Null Value Representation": { "name": "Null Value Representation", "displayName": "Null Value Representation", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "station_uuid": { "name": "station_uuid", "displayName": "station_uuid", "identifiesControllerService": false, "sensitive": false, "dynamic": true }, "Path Not Found Behavior": { "name": "Path Not Found Behavior", "displayName": "Path Not Found Behavior", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "failure", "unmatched" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "99d1a93c-6332-3e77-0000-0000527115be", "instanceIdentifier": "839f75a2-3a32-38b7-8c36-50498002e9c1", "name": "PublishKafka", "comments": "", "position": { "x": 936.0, "y": 1432.0 }, "type": "org.apache.nifi.kafka.processors.PublishKafka", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-nar", "version": "2.6.0" }, "properties": { "compression.type": "snappy", "acks": "all", "Message Demarcator": null, "Kafka Key Attribute Encoding": "utf-8", "Record Reader": "c127588a-e95f-382f-afb1-33f9b279c38d", "Record Metadata Strategy": "FROM_PROPERTIES", "Transactional ID Prefix": null, "Header Encoding": "UTF-8", "max.request.size": "1 MB", "Topic Name": "water_levels_measurements", "Kafka Key": null, "partition": null, "Kafka Connection Service": "a7cc755f-2210-3763-889e-24c3e0b0684d", "Publish Strategy": "USE_VALUE", "Record Key Writer": null, "Failure Strategy": "Route to Failure", "partitioner.class": "org.apache.kafka.clients.producer.internals.DefaultPartitioner", "Record Writer": "50e96367-7b84-3034-8dbf-6e02fa7c01df", "Transactions Enabled": "false", "Message Key Field": null, "FlowFile Attribute Header Pattern": null }, "propertyDescriptors": { "compression.type": { "name": "compression.type", "displayName": "Compression Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "acks": { "name": "acks", "displayName": "Delivery Guarantee", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Demarcator": { "name": "Message Demarcator", "displayName": "Message Demarcator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key Attribute Encoding": { "name": "Kafka Key Attribute Encoding", "displayName": "Kafka Key Attribute Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Metadata Strategy": { "name": "Record Metadata Strategy", "displayName": "Record Metadata Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transactional ID Prefix": { "name": "Transactional ID Prefix", "displayName": "Transactional ID Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Header Encoding": { "name": "Header Encoding", "displayName": "Header Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.request.size": { "name": "max.request.size", "displayName": "Max Request Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Topic Name": { "name": "Topic Name", "displayName": "Topic Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key": { "name": "Kafka Key", "displayName": "Kafka Key", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partition": { "name": "partition", "displayName": "Partition", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Connection Service": { "name": "Kafka Connection Service", "displayName": "Kafka Connection Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Publish Strategy": { "name": "Publish Strategy", "displayName": "Publish Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Key Writer": { "name": "Record Key Writer", "displayName": "Record Key Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Failure Strategy": { "name": "Failure Strategy", "displayName": "Failure Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partitioner.class": { "name": "partitioner.class", "displayName": "Partitioner Class", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Transactions Enabled": { "name": "Transactions Enabled", "displayName": "Transactions Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Key Field": { "name": "Message Key Field", "displayName": "Message Key Field", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "FlowFile Attribute Header Pattern": { "name": "FlowFile Attribute Header Pattern", "displayName": "FlowFile Attribute Header Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "success", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "762fb4f3-201b-3c92-ffff-ffff8e7e57fd", "instanceIdentifier": "3d7eb954-e054-3324-aea1-5aa7a03dc120", "name": "PublishKafka", "comments": "", "position": { "x": 1312.0, "y": 1432.0 }, "type": "org.apache.nifi.kafka.processors.PublishKafka", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-nar", "version": "2.6.0" }, "properties": { "compression.type": "snappy", "acks": "all", "Message Demarcator": null, "Kafka Key Attribute Encoding": "utf-8", "Record Reader": "c127588a-e95f-382f-afb1-33f9b279c38d", "Record Metadata Strategy": "FROM_PROPERTIES", "Transactional ID Prefix": null, "Header Encoding": "UTF-8", "max.request.size": "1 MB", "Topic Name": "water_levels_measurements", "Kafka Key": null, "partition": null, "Kafka Connection Service": "a7cc755f-2210-3763-889e-24c3e0b0684d", "Publish Strategy": "USE_VALUE", "Record Key Writer": null, "Failure Strategy": "Route to Failure", "partitioner.class": "org.apache.kafka.clients.producer.internals.DefaultPartitioner", "Record Writer": "50e96367-7b84-3034-8dbf-6e02fa7c01df", "Transactions Enabled": "false", "Message Key Field": null, "FlowFile Attribute Header Pattern": null }, "propertyDescriptors": { "compression.type": { "name": "compression.type", "displayName": "Compression Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "acks": { "name": "acks", "displayName": "Delivery Guarantee", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Demarcator": { "name": "Message Demarcator", "displayName": "Message Demarcator", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key Attribute Encoding": { "name": "Kafka Key Attribute Encoding", "displayName": "Kafka Key Attribute Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Metadata Strategy": { "name": "Record Metadata Strategy", "displayName": "Record Metadata Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Transactional ID Prefix": { "name": "Transactional ID Prefix", "displayName": "Transactional ID Prefix", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Header Encoding": { "name": "Header Encoding", "displayName": "Header Encoding", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.request.size": { "name": "max.request.size", "displayName": "Max Request Size", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Topic Name": { "name": "Topic Name", "displayName": "Topic Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Key": { "name": "Kafka Key", "displayName": "Kafka Key", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partition": { "name": "partition", "displayName": "Partition", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Kafka Connection Service": { "name": "Kafka Connection Service", "displayName": "Kafka Connection Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Publish Strategy": { "name": "Publish Strategy", "displayName": "Publish Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Key Writer": { "name": "Record Key Writer", "displayName": "Record Key Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Failure Strategy": { "name": "Failure Strategy", "displayName": "Failure Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "partitioner.class": { "name": "partitioner.class", "displayName": "Partitioner Class", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Transactions Enabled": { "name": "Transactions Enabled", "displayName": "Transactions Enabled", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Message Key Field": { "name": "Message Key Field", "displayName": "Message Key Field", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "FlowFile Attribute Header Pattern": { "name": "FlowFile Attribute Header Pattern", "displayName": "FlowFile Attribute Header Pattern", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "success", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "c6929ff2-ea95-3d79-a483-3ffa75371ee4", "instanceIdentifier": "78b9af30-7922-3dd6-91d4-f88d7f984ec6", "name": "SplitRecord", "comments": "", "position": { "x": 933.000000762879, "y": 537.999988448508 }, "type": "org.apache.nifi.processors.standard.SplitRecord", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-nar", "version": "2.6.0" }, "properties": { "Record Writer": "50e96367-7b84-3034-8dbf-6e02fa7c01df", "Record Reader": "c127588a-e95f-382f-afb1-33f9b279c38d", "Records Per Split": "1" }, "propertyDescriptors": { "Record Writer": { "name": "Record Writer", "displayName": "Record Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Record Reader": { "name": "Record Reader", "displayName": "Record Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Records Per Split": { "name": "Records Per Split", "displayName": "Records Per Split", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "style": {}, "schedulingPeriod": "0 sec", "schedulingStrategy": "TIMER_DRIVEN", "executionNode": "ALL", "penaltyDuration": "30 sec", "yieldDuration": "1 sec", "bulletinLevel": "WARN", "runDurationMillis": 0, "concurrentlySchedulableTaskCount": 1, "autoTerminatedRelationships": [ "original", "failure" ], "scheduledState": "ENABLED", "retryCount": 10, "retriedRelationships": [], "backoffMechanism": "PENALIZE_FLOWFILE", "maxBackoffPeriod": "10 mins", "componentType": "PROCESSOR", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" } ], "inputPorts": [], "outputPorts": [], "connections": [ { "identifier": "9552346e-6dd7-3991-99f5-479e87ebb4b6", "instanceIdentifier": "1bfdc782-8410-3e9d-a004-7d21c43fa5bb", "name": "", "source": { "id": "26a592be-7444-337c-b8f9-3a29013cb760", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Get station status", "comments": "", "instanceIdentifier": "0ea082f4-3f0d-38db-bd4b-51feb5428cc6" }, "destination": { "id": "3a46b8c1-c2c6-3006-0000-000066d6dbdd", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Extract data attribute", "comments": "", "instanceIdentifier": "9e653a1f-9897-3124-8aa1-83321219f2fc" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 10000, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "2c1c00a7-bf5d-3937-8fb3-8ee54b5117e3", "instanceIdentifier": "eb0cef96-08d3-39de-8518-cf136ccd5871", "name": "", "source": { "id": "5e50831e-a2fd-34c9-af0a-924b16280705", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Get station information", "comments": "", "instanceIdentifier": "54f2ba99-3a49-3160-b8b9-45b8c0a6ac2a" }, "destination": { "id": "405f050c-5005-3b96-ffff-ffffbc8a379e", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Extract data attribute", "comments": "", "instanceIdentifier": "3dd55349-9324-36ea-a6a0-d76280b79609" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 10000, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "e7f33855-2961-3414-8c28-3056fcdac571", "instanceIdentifier": "dc784d10-a696-3f10-94b8-f6bb1fae7235", "name": "", "source": { "id": "00bcf948-7bef-35b6-8a6d-e641194b8189", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Get station list", "comments": "", "instanceIdentifier": "d92196ba-38f9-3b77-b9a4-4e0dfe564d3f" }, "destination": { "id": "1a0aea35-946c-35de-8811-549faeb003ee", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Produce station records", "comments": "", "instanceIdentifier": "81675904-0195-1000-ffff-ffffbbbb9cfe" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 10000, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "f1255a8c-06e1-310c-bd02-5e34b218a02f", "instanceIdentifier": "8172e290-0195-1000-0000-0000188da43e", "name": "", "source": { "id": "056f4faa-a4dc-3024-ffff-ffffb73ef3a0", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Extract data attribute", "comments": "", "instanceIdentifier": "7b0b12f3-669e-32c4-b866-dea8a4a6f34a" }, "destination": { "id": "d9bacb7c-10df-3044-ffff-ffffe7a604af", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Produce bike status records", "comments": "", "instanceIdentifier": "8c7c157b-6f95-399c-83f7-54a79ea11078" }, "labelIndex": 0, "zIndex": 0, "selectedRelationships": [ "success" ], "backPressureObjectThreshold": 10000, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "ae09a6b6-1dcd-35bc-82ec-62171973c61b", "instanceIdentifier": "a742154d-a24c-3bb8-9f65-cbc95810a284", "name": "", "source": { "id": "de71001e-de69-32c2-82b9-6f008665b252", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Get 30 minutes of historic data", "comments": "", "instanceIdentifier": "df138016-7764-3ad6-ac10-9f57e657461d" }, "destination": { "id": "42346af3-5793-3b8c-ffff-ffffc5c5139d", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Add station_uuid", "comments": "", "instanceIdentifier": "f9f95954-77fc-3f16-9b94-54ca68090b21" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 10, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "709e7c78-6123-386a-a0ce-5a7394c4d39e", "instanceIdentifier": "9ab4b88c-e4c3-3303-88d1-18c8b5f00817", "name": "", "source": { "id": "fc594301-ecc4-353b-8330-7aecb14e6422", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Get station list", "comments": "", "instanceIdentifier": "f049f0b7-46f9-332b-bce3-925167b05f37" }, "destination": { "id": "577f603d-ee9c-3202-8139-d35d94c9c311", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "SplitRecord", "comments": "", "instanceIdentifier": "331d7feb-96a3-3998-9a1e-5b23cca85c5f" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 1, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "e4d5da00-e103-3166-8381-d4f2533c7a9d", "instanceIdentifier": "a559d40e-de21-3d84-a6e8-1abcfe13601a", "name": "", "source": { "id": "7133f561-971a-35b6-abff-9892e06b4bbe", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Get 30 days historic data", "comments": "", "instanceIdentifier": "fd7775f9-0f65-361a-bcb9-71668ae1a663" }, "destination": { "id": "ba73d7c1-102f-3888-a573-5e6d2ec9e1f0", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Add station_uuid", "comments": "", "instanceIdentifier": "816f11f1-0195-1000-ffff-ffffd698d6c2" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 10, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "fd7e3f54-ea73-3f6b-9700-33442d5ee235", "instanceIdentifier": "72965aa1-df12-31a6-9eb8-ac5d006406e0", "name": "", "source": { "id": "577f603d-ee9c-3202-8139-d35d94c9c311", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "SplitRecord", "comments": "", "instanceIdentifier": "331d7feb-96a3-3998-9a1e-5b23cca85c5f" }, "destination": { "id": "0640256b-010a-3e69-b961-680e63ec85e2", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Extract station_uuid", "comments": "", "instanceIdentifier": "71fee077-9f08-3916-b189-f933b8f13619" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "splits" ], "backPressureObjectThreshold": 10, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "ROUND_ROBIN", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "3bec8550-723b-3b2c-89c9-d57298a1ab03", "instanceIdentifier": "816ffee3-0195-1000-ffff-ffffd4be0c57", "name": "", "source": { "id": "ba73d7c1-102f-3888-a573-5e6d2ec9e1f0", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Add station_uuid", "comments": "", "instanceIdentifier": "816f11f1-0195-1000-ffff-ffffd698d6c2" }, "destination": { "id": "99d1a93c-6332-3e77-0000-0000527115be", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "PublishKafka", "comments": "", "instanceIdentifier": "839f75a2-3a32-38b7-8c36-50498002e9c1" }, "labelIndex": 0, "zIndex": 0, "selectedRelationships": [ "success" ], "backPressureObjectThreshold": 20, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "14af6270-d836-3045-a5ca-1e699f1da8ee", "instanceIdentifier": "81729813-0195-1000-0000-0000137d67da", "name": "", "source": { "id": "3a46b8c1-c2c6-3006-0000-000066d6dbdd", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Extract data attribute", "comments": "", "instanceIdentifier": "9e653a1f-9897-3124-8aa1-83321219f2fc" }, "destination": { "id": "dd4690f7-ae51-36bb-0000-0000325e1eed", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Produce station status records", "comments": "", "instanceIdentifier": "664aded4-922a-3da4-a584-7d5e43361201" }, "labelIndex": 0, "zIndex": 0, "selectedRelationships": [ "success" ], "backPressureObjectThreshold": 10000, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "138e8f4a-3dbd-3b9a-a259-c00bd60194c1", "instanceIdentifier": "796cfe4a-0f27-38df-a07f-03c8f172cc47", "name": "", "source": { "id": "c6929ff2-ea95-3d79-a483-3ffa75371ee4", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "SplitRecord", "comments": "", "instanceIdentifier": "78b9af30-7922-3dd6-91d4-f88d7f984ec6" }, "destination": { "id": "a1453322-1c3f-3150-a5be-0330c98144d9", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Extract station_uuid", "comments": "", "instanceIdentifier": "be5c2f57-09fd-3608-8c9c-9779e52c0ce8" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "splits" ], "backPressureObjectThreshold": 10, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "ROUND_ROBIN", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "d09052a3-4cb1-35c1-bed3-cbee709b53b3", "instanceIdentifier": "84221825-bfbc-3c5b-82bb-bfc3e21fcb48", "name": "", "source": { "id": "a1453322-1c3f-3150-a5be-0330c98144d9", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Extract station_uuid", "comments": "", "instanceIdentifier": "be5c2f57-09fd-3608-8c9c-9779e52c0ce8" }, "destination": { "id": "7133f561-971a-35b6-abff-9892e06b4bbe", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Get 30 days historic data", "comments": "", "instanceIdentifier": "fd7775f9-0f65-361a-bcb9-71668ae1a663" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "matched" ], "backPressureObjectThreshold": 10, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "572214c6-71ac-3533-964f-fd44e1ec17f2", "instanceIdentifier": "8171dd08-0195-1000-0000-000061d1719e", "name": "", "source": { "id": "405f050c-5005-3b96-ffff-ffffbc8a379e", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Extract data attribute", "comments": "", "instanceIdentifier": "3dd55349-9324-36ea-a6a0-d76280b79609" }, "destination": { "id": "90d02439-1516-3c8a-ffff-ffff8b2ffb20", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Produce station information records", "comments": "", "instanceIdentifier": "3a5f9f2c-bafa-3ae3-bc1b-2e8f1dbea59e" }, "labelIndex": 0, "zIndex": 0, "selectedRelationships": [ "success" ], "backPressureObjectThreshold": 10000, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "c75b4056-44a4-3f5e-93f8-e461b2550623", "instanceIdentifier": "373d1546-8ba2-3b9a-9c66-6b8f9b1917b8", "name": "", "source": { "id": "364e250b-f224-3e12-b4cf-c782c9a06bbe", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Get free bike status", "comments": "", "instanceIdentifier": "143b2ea1-f9cc-3140-afd1-426a846dd9d0" }, "destination": { "id": "056f4faa-a4dc-3024-ffff-ffffb73ef3a0", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Extract data attribute", "comments": "", "instanceIdentifier": "7b0b12f3-669e-32c4-b866-dea8a4a6f34a" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 10000, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "5448c0cf-2aa5-3ef8-8ecd-836ab3fdb853", "instanceIdentifier": "0eff3813-31ae-3e6e-9f91-8883c0b83ff4", "name": "", "source": { "id": "98c83f54-9f99-3d95-88b0-ae91140707c9", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Get station list", "comments": "", "instanceIdentifier": "098befce-068e-3d63-8586-1a7309e40e58" }, "destination": { "id": "c6929ff2-ea95-3d79-a483-3ffa75371ee4", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "SplitRecord", "comments": "", "instanceIdentifier": "78b9af30-7922-3dd6-91d4-f88d7f984ec6" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "Response" ], "backPressureObjectThreshold": 1, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "9acf6812-3d3a-3f35-94e9-fa6eb209342f", "instanceIdentifier": "8170bfea-0195-1000-ffff-ffffdc31cffd", "name": "", "source": { "id": "42346af3-5793-3b8c-ffff-ffffc5c5139d", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Add station_uuid", "comments": "", "instanceIdentifier": "f9f95954-77fc-3f16-9b94-54ca68090b21" }, "destination": { "id": "762fb4f3-201b-3c92-ffff-ffff8e7e57fd", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "PublishKafka", "comments": "", "instanceIdentifier": "3d7eb954-e054-3324-aea1-5aa7a03dc120" }, "labelIndex": 0, "zIndex": 0, "selectedRelationships": [ "success" ], "backPressureObjectThreshold": 20, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "1ca5e75c-5900-3f28-94cb-25f1ade9a5a0", "instanceIdentifier": "ee452230-0ad0-3409-8b03-39acaf8ec15e", "name": "", "source": { "id": "0640256b-010a-3e69-b961-680e63ec85e2", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Extract station_uuid", "comments": "", "instanceIdentifier": "71fee077-9f08-3916-b189-f933b8f13619" }, "destination": { "id": "de71001e-de69-32c2-82b9-6f008665b252", "type": "PROCESSOR", "groupId": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911", "name": "Get 30 minutes of historic data", "comments": "", "instanceIdentifier": "df138016-7764-3ad6-ac10-9f57e657461d" }, "labelIndex": 1, "zIndex": 0, "selectedRelationships": [ "matched" ], "backPressureObjectThreshold": 10, "backPressureDataSizeThreshold": "1 GB", "flowFileExpiration": "0 sec", "prioritizers": [], "bends": [], "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", "partitioningAttribute": "", "loadBalanceCompression": "DO_NOT_COMPRESS", "componentType": "CONNECTION", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" } ], "labels": [ { "identifier": "f531aeda-2cad-3391-bc64-d10934a5f355", "instanceIdentifier": "f990fed8-9ffc-3b20-8795-efeaf1c55396", "position": { "x": 1309.0000007628792, "y": 241.99998844850802 }, "label": "Stream real-time data", "zIndex": 0, "width": 152.0, "height": 24.0, "style": { "font-size": "12px" }, "componentType": "LABEL", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "b4bf22b0-ff9e-3244-90bd-c6306a8e1fe5", "instanceIdentifier": "fdd434de-64f4-36e8-b1a8-63ad26549901", "position": { "x": 1701.0000007628792, "y": 201.99998844850802 }, "label": "Shared bikes data", "zIndex": 0, "width": 1152.0, "height": 24.0, "style": { "font-size": "12px" }, "componentType": "LABEL", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "637f34a6-0a08-34a9-bdcd-b3bfc4257a4c", "instanceIdentifier": "730910f3-60f8-3278-9098-5d864bca1ad3", "position": { "x": 2501.000000762879, "y": 241.99998844850802 }, "label": "Ingest bike status", "zIndex": 0, "width": 152.0, "height": 24.0, "style": { "font-size": "12px" }, "componentType": "LABEL", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "cb5fcdeb-c972-34cb-af5a-f6812416ed2d", "instanceIdentifier": "aafc5a3c-bbc6-3ef0-81e8-743f9dd99644", "position": { "x": 1701.0000007628792, "y": 241.99998844850802 }, "label": "Ingest station information", "zIndex": 0, "width": 152.0, "height": 24.0, "style": { "font-size": "12px" }, "componentType": "LABEL", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "f3bd6d3a-3dea-3909-a60e-aa3a34542e5c", "instanceIdentifier": "b8ec73a5-19b4-3d53-b152-7d99129c6342", "position": { "x": 557.000000762879, "y": 241.99998844850802 }, "label": "Ingest station records", "zIndex": 0, "width": 152.0, "height": 24.0, "style": { "font-size": "12px" }, "componentType": "LABEL", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "6dcd3b2a-e51d-329f-9aca-bffdc414bb51", "instanceIdentifier": "61a589ce-b17b-3090-ae4b-e9ce0dab0047", "position": { "x": 2101.000000762879, "y": 241.99998844850802 }, "label": "Ingest station status", "zIndex": 0, "width": 152.0, "height": 24.0, "style": { "font-size": "12px" }, "componentType": "LABEL", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "543a323b-3976-313c-9344-05c172ab5d02", "instanceIdentifier": "8e99d259-10ae-3f12-8b51-fdca63c2c201", "position": { "x": 933.000000762879, "y": 241.99998844850802 }, "label": "Ingest historic data", "zIndex": 0, "width": 152.0, "height": 24.0, "style": { "font-size": "12px" }, "componentType": "LABEL", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "f9ea8997-f618-3472-9745-93ca1036d176", "instanceIdentifier": "0393438f-a1a3-3b2a-8012-66c5050636cd", "position": { "x": 557.000000762879, "y": 201.99998844850802 }, "label": "Water levels data", "zIndex": 0, "width": 1104.0, "height": 24.0, "style": { "font-size": "12px" }, "componentType": "LABEL", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" } ], "funnels": [], "controllerServices": [ { "identifier": "906147e9-0ba3-3951-8764-15e14a9766d6", "instanceIdentifier": "31167665-331b-3abd-bbec-8026382f3e57", "name": "StandardRestrictedSSLContextService", "comments": "", "type": "org.apache.nifi.ssl.StandardRestrictedSSLContextService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-ssl-context-service-nar", "version": "2.6.0" }, "properties": { "Truststore Type": "PKCS12", "Truststore Password": "secret", "SSL Protocol": "TLSv1.3", "Keystore Password": "secret", "Keystore Type": "PKCS12", "Truststore Filename": "/stackable/keystore/truststore.p12", "Keystore Filename": "/stackable/keystore/keystore.p12" }, "propertyDescriptors": { "Truststore Type": { "name": "Truststore Type", "displayName": "Truststore Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Protocol": { "name": "SSL Protocol", "displayName": "TLS Protocol", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Keystore Type": { "name": "Keystore Type", "displayName": "Keystore Type", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Truststore Filename": { "name": "Truststore Filename", "displayName": "Truststore Filename", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "SINGLE", "resourceTypes": [ "FILE" ] } }, "Keystore Password": { "name": "Keystore Password", "displayName": "Keystore Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "key-password": { "name": "key-password", "displayName": "Key Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Truststore Password": { "name": "Truststore Password", "displayName": "Truststore Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "Keystore Filename": { "name": "Keystore Filename", "displayName": "Keystore Filename", "identifiesControllerService": false, "sensitive": false, "dynamic": false, "resourceDefinition": { "cardinality": "SINGLE", "resourceTypes": [ "FILE" ] } } }, "controllerServiceApis": [ { "type": "org.apache.nifi.ssl.SSLContextProvider", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } }, { "type": "org.apache.nifi.ssl.SSLContextService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } }, { "type": "org.apache.nifi.ssl.RestrictedSSLContextService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } } ], "scheduledState": "DISABLED", "bulletinLevel": "WARN", "componentType": "CONTROLLER_SERVICE", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "50e96367-7b84-3034-8dbf-6e02fa7c01df", "instanceIdentifier": "d795c667-0833-34e1-9d2b-4ae37d102704", "name": "JsonRecordSetWriter", "type": "org.apache.nifi.json.JsonRecordSetWriter", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-record-serialization-services-nar", "version": "2.6.0" }, "properties": { "schema-reference-reader": null, "schema-branch": null, "Allow Scientific Notation": "false", "compression-level": "1", "schema-cache": null, "Timestamp Format": null, "Date Format": null, "Pretty Print JSON": "false", "compression-format": "none", "Schema Write Strategy": "no-schema", "suppress-nulls": "never-suppress", "output-grouping": "output-array", "schema-name": "${schema.name}", "schema-registry": null, "Time Format": null, "schema-access-strategy": "inherit-record-schema", "schema-version": null, "schema-text": "${avro.schema}", "Schema Reference Writer": null }, "propertyDescriptors": { "schema-reference-reader": { "name": "schema-reference-reader", "displayName": "Schema Reference Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "schema-branch": { "name": "schema-branch", "displayName": "Schema Branch", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Allow Scientific Notation": { "name": "Allow Scientific Notation", "displayName": "Allow Scientific Notation", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "compression-level": { "name": "compression-level", "displayName": "Compression Level", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-cache": { "name": "schema-cache", "displayName": "Schema Cache", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Timestamp Format": { "name": "Timestamp Format", "displayName": "Timestamp Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Date Format": { "name": "Date Format", "displayName": "Date Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Pretty Print JSON": { "name": "Pretty Print JSON", "displayName": "Pretty Print JSON", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "compression-format": { "name": "compression-format", "displayName": "Compression Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Schema Write Strategy": { "name": "Schema Write Strategy", "displayName": "Schema Write Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "suppress-nulls": { "name": "suppress-nulls", "displayName": "Suppress Null Values", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "output-grouping": { "name": "output-grouping", "displayName": "Output Grouping", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-name": { "name": "schema-name", "displayName": "Schema Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-registry": { "name": "schema-registry", "displayName": "Schema Registry", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Time Format": { "name": "Time Format", "displayName": "Time Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-access-strategy": { "name": "schema-access-strategy", "displayName": "Schema Access Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-version": { "name": "schema-version", "displayName": "Schema Version", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-text": { "name": "schema-text", "displayName": "Schema Text", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Schema Reference Writer": { "name": "Schema Reference Writer", "displayName": "Schema Reference Writer", "identifiesControllerService": true, "sensitive": false, "dynamic": false } }, "controllerServiceApis": [ { "type": "org.apache.nifi.serialization.RecordSetWriterFactory", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } } ], "scheduledState": "DISABLED", "bulletinLevel": "WARN", "componentType": "CONTROLLER_SERVICE", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "a7cc755f-2210-3763-889e-24c3e0b0684d", "instanceIdentifier": "8167b5da-0195-1000-ffff-ffff926c765c", "name": "Kafka3ConnectionService", "comments": "", "type": "org.apache.nifi.kafka.service.Kafka3ConnectionService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-3-service-nar", "version": "2.6.0" }, "properties": { "security.protocol": "SSL", "SSL Context Service": "906147e9-0ba3-3951-8764-15e14a9766d6", "ack.wait.time": "5 sec", "sasl.username": null, "max.poll.records": "10000", "sasl.mechanism": "GSSAPI", "isolation.level": "read_committed", "max.block.ms": "5 sec", "bootstrap.servers": "kafka-broker-default-headless.{{ NAMESPACE }}.svc.cluster.local:9093", "kerberos-user-service": null, "sasl.kerberos.service.name": null, "default.api.timeout.ms": "60 sec" }, "propertyDescriptors": { "ack.wait.time": { "name": "ack.wait.time", "displayName": "Acknowledgment Wait Time", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "sasl.username": { "name": "sasl.username", "displayName": "SASL Username", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.poll.records": { "name": "max.poll.records", "displayName": "Max Poll Records", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "bootstrap.servers": { "name": "bootstrap.servers", "displayName": "Bootstrap Servers", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "sasl.kerberos.service.name": { "name": "sasl.kerberos.service.name", "displayName": "Kerberos Service Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "security.protocol": { "name": "security.protocol", "displayName": "Security Protocol", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "SSL Context Service": { "name": "SSL Context Service", "displayName": "SSL Context Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "sasl.mechanism": { "name": "sasl.mechanism", "displayName": "SASL Mechanism", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "isolation.level": { "name": "isolation.level", "displayName": "Transaction Isolation Level", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "max.block.ms": { "name": "max.block.ms", "displayName": "Max Metadata Wait Time", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "kerberos-user-service": { "name": "kerberos-user-service", "displayName": "Kerberos User Service", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "sasl.password": { "name": "sasl.password", "displayName": "SASL Password", "identifiesControllerService": false, "sensitive": true, "dynamic": false }, "default.api.timeout.ms": { "name": "default.api.timeout.ms", "displayName": "Client Timeout", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "controllerServiceApis": [ { "type": "org.apache.nifi.kafka.service.api.KafkaConnectionService", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-kafka-service-api-nar", "version": "2.6.0" } } ], "scheduledState": "DISABLED", "bulletinLevel": "WARN", "componentType": "CONTROLLER_SERVICE", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" }, { "identifier": "c127588a-e95f-382f-afb1-33f9b279c38d", "instanceIdentifier": "715ff6c2-fd41-366f-9922-e0365c063f33", "name": "JsonTreeReader", "type": "org.apache.nifi.json.JsonTreeReader", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-record-serialization-services-nar", "version": "2.6.0" }, "properties": { "schema-reference-reader": null, "schema-branch": null, "Max String Length": "20 MB", "schema-application-strategy": "SELECTED_PART", "Timestamp Format": null, "schema-inference-cache": null, "Date Format": null, "schema-name": "${schema.name}", "starting-field-strategy": "ROOT_NODE", "schema-registry": null, "starting-field-name": null, "Time Format": null, "schema-access-strategy": "infer-schema", "schema-version": null, "schema-text": "${avro.schema}", "Allow Comments": "false" }, "propertyDescriptors": { "schema-reference-reader": { "name": "schema-reference-reader", "displayName": "Schema Reference Reader", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "schema-branch": { "name": "schema-branch", "displayName": "Schema Branch", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Max String Length": { "name": "Max String Length", "displayName": "Max String Length", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-application-strategy": { "name": "schema-application-strategy", "displayName": "Schema Application Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Timestamp Format": { "name": "Timestamp Format", "displayName": "Timestamp Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-inference-cache": { "name": "schema-inference-cache", "displayName": "Schema Inference Cache", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "Date Format": { "name": "Date Format", "displayName": "Date Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-name": { "name": "schema-name", "displayName": "Schema Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "starting-field-strategy": { "name": "starting-field-strategy", "displayName": "Starting Field Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-registry": { "name": "schema-registry", "displayName": "Schema Registry", "identifiesControllerService": true, "sensitive": false, "dynamic": false }, "starting-field-name": { "name": "starting-field-name", "displayName": "Starting Field Name", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Time Format": { "name": "Time Format", "displayName": "Time Format", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-access-strategy": { "name": "schema-access-strategy", "displayName": "Schema Access Strategy", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-version": { "name": "schema-version", "displayName": "Schema Version", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "schema-text": { "name": "schema-text", "displayName": "Schema Text", "identifiesControllerService": false, "sensitive": false, "dynamic": false }, "Allow Comments": { "name": "Allow Comments", "displayName": "Allow Comments", "identifiesControllerService": false, "sensitive": false, "dynamic": false } }, "controllerServiceApis": [ { "type": "org.apache.nifi.serialization.RecordReaderFactory", "bundle": { "group": "org.apache.nifi", "artifact": "nifi-standard-services-api-nar", "version": "2.6.0" } } ], "scheduledState": "DISABLED", "bulletinLevel": "WARN", "componentType": "CONTROLLER_SERVICE", "groupIdentifier": "eaab7dc0-4761-3f1f-9dd5-ff82bd301911" } ], "defaultFlowFileExpiration": "0 sec", "defaultBackPressureObjectThreshold": 10000, "defaultBackPressureDataSizeThreshold": "1 GB", "scheduledState": "ENABLED", "executionEngine": "INHERITED", "maxConcurrentTasks": 1, "statelessFlowTimeout": "1 min", "flowFileConcurrency": "UNBOUNDED", "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE", "componentType": "PROCESS_GROUP" }, "externalControllerServices": {}, "parameterContexts": {}, "flowEncodingVersion": "1.0", "parameterProviders": {}, "latest": false } diff --git a/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml b/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml index 8751e9f6..3e6e8408 100644 --- a/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml +++ b/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml @@ -20,11 +20,11 @@ spec: - -c - | echo 'Waiting for all minio instances to be ready' - kubectl wait --for=condition=ready --timeout=30m pod -l app=minio,release=minio,stackable.tech/vendor=Stackable + kubectl wait --for=condition=ready --timeout=30m pod -l app=minio,release=minio,stackable.tech/vendor=Stackable -n {{ NAMESPACE }} echo 'Waiting for all kafka brokers to be ready' - kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/name=kafka,app.kubernetes.io/instance=kafka + kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/name=kafka,app.kubernetes.io/instance=kafka -n {{ NAMESPACE }} echo 'Waiting for all nifi instances to be ready' - kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/name=nifi,app.kubernetes.io/instance=nifi + kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/name=nifi,app.kubernetes.io/instance=nifi -n {{ NAMESPACE }} - name: wait-for-kafka-topics image: oci.stackable.tech/sdp/kafka:4.1.0-stackable0.0.0-dev command: @@ -142,6 +142,7 @@ data: name: spark-ingest-into-lakehouse labels: stackable.tech/vendor: Stackable + stackable.tech/demo: {{ demoLabel }} spec: sparkImage: productVersion: 4.0.1 @@ -253,7 +254,7 @@ data: spark.sql("CREATE TABLE IF NOT EXISTS lakehouse.smart_city.shared_bikes_station_status (station_id string, num_bikes_available short, is_installed boolean, is_renting boolean, is_returning boolean, last_reported timestamp) USING iceberg PARTITIONED BY (days(last_reported)) TBLPROPERTIES ('format-version' = 2, format = 'PARQUET')") kafkaOptions = { - "kafka.bootstrap.servers": "kafka-broker-default.default.svc.cluster.local:9093", + "kafka.bootstrap.servers": "kafka-broker-default.{{ NAMESPACE }}.svc.cluster.local:9093", "kafka.security.protocol": "SSL", "kafka.ssl.truststore.location": "/stackable/tls/truststore.p12", "kafka.ssl.truststore.password": "changeit", diff --git a/demos/data-lakehouse-iceberg-trino-spark/create-trino-tables.yaml b/demos/data-lakehouse-iceberg-trino-spark/create-trino-tables.yaml index 918cad7e..8e3ef366 100644 --- a/demos/data-lakehouse-iceberg-trino-spark/create-trino-tables.yaml +++ b/demos/data-lakehouse-iceberg-trino-spark/create-trino-tables.yaml @@ -17,7 +17,7 @@ spec: - -c - | echo 'Waiting for job load-test-data to finish' - kubectl wait --for=condition=complete --timeout=30m job/load-test-data + kubectl wait --for=condition=complete --timeout=30m job/load-test-data -n {{ NAMESPACE }} containers: - name: create-tables-in-trino image: oci.stackable.tech/sdp/testing-tools/trino:0.3.0-stackable0.0.0-dev diff --git a/demos/data-lakehouse-iceberg-trino-spark/load-test-data.yaml b/demos/data-lakehouse-iceberg-trino-spark/load-test-data.yaml index 1f810ba0..a8e2a028 100644 --- a/demos/data-lakehouse-iceberg-trino-spark/load-test-data.yaml +++ b/demos/data-lakehouse-iceberg-trino-spark/load-test-data.yaml @@ -19,7 +19,7 @@ spec: # Copy the CA cert from the "tls" SecretClass cp -v /etc/minio/mc/original_certs/ca.crt /.mc/certs/CAs/public.crt - mc alias set minio https://minio.default.svc.cluster.local:9000/ $(cat /minio-s3-credentials/accessKey) $(cat /minio-s3-credentials/secretKey) + mc alias set minio https://minio.{{ NAMESPACE }}.svc.cluster.local:9000/ $(cat /minio-s3-credentials/accessKey) $(cat /minio-s3-credentials/secretKey) cd /tmp curl -sO https://repo.stackable.tech/repository/misc/datasets/open-postcode-geo/open-postcode-geo.csv diff --git a/demos/data-lakehouse-iceberg-trino-spark/serviceaccount.yaml b/demos/data-lakehouse-iceberg-trino-spark/serviceaccount.yaml index 53e2ccae..5b50b941 100644 --- a/demos/data-lakehouse-iceberg-trino-spark/serviceaccount.yaml +++ b/demos/data-lakehouse-iceberg-trino-spark/serviceaccount.yaml @@ -3,7 +3,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: demo-serviceaccount - namespace: default --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -12,7 +11,7 @@ metadata: subjects: - kind: ServiceAccount name: demo-serviceaccount - namespace: default + namespace: {{ NAMESPACE }} roleRef: kind: ClusterRole name: demo-clusterrole diff --git a/demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml b/demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml index bda7a2a9..2325cfb8 100644 --- a/demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml +++ b/demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml @@ -15,6 +15,7 @@ spec: - pipefail - -c - | + # This file path needs adjustment for versioned demos on release branch curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/superset-assets.zip python -u /tmp/script/script.py volumeMounts: diff --git a/demos/demos-v2.yaml b/demos/demos-v2.yaml index 79efc605..a1d3bc3e 100644 --- a/demos/demos-v2.yaml +++ b/demos/demos-v2.yaml @@ -218,11 +218,15 @@ demos: - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml - supportedNamespaces: ["default"] + supportedNamespaces: [] resourceRequests: cpu: "80" memory: 200Gi pvc: 300Gi + parameters: + - name: demoLabel + description: Demo label to add to various components + default: data-lakehouse-iceberg-trino-spark jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data: description: Jupyterhub with PySpark and HDFS integration documentation: https://docs.stackable.tech/home/stable/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data diff --git a/stacks/_templates/minio-distributed-tls/rendered-chart.yaml b/stacks/_templates/minio-distributed-tls/rendered-chart.yaml index 4a5b2cfc..249048b5 100644 --- a/stacks/_templates/minio-distributed-tls/rendered-chart.yaml +++ b/stacks/_templates/minio-distributed-tls/rendered-chart.yaml @@ -541,7 +541,7 @@ spec: cp -v /etc/minio/original_certs/tls.key /etc/minio/certs/private.key cp -v /etc/minio/original_certs/ca.crt /etc/minio/certs/CAs/ca.crt - /usr/bin/docker-entrypoint.sh minio server https://minio-{0...4}.minio-svc.default.svc.cluster.local/export -S /etc/minio/certs/ --address :9000 --console-address :9001 + /usr/bin/docker-entrypoint.sh minio server https://minio-{0...4}.minio-svc.{{ NAMESPACE }}.svc.cluster.local/export -S /etc/minio/certs/ --address :9000 --console-address :9001 volumeMounts: - name: export mountPath: /export diff --git a/stacks/_templates/minio-distributed-tls/values.yaml b/stacks/_templates/minio-distributed-tls/values.yaml index 95da9110..570e9748 100644 --- a/stacks/_templates/minio-distributed-tls/values.yaml +++ b/stacks/_templates/minio-distributed-tls/values.yaml @@ -1,7 +1,9 @@ additionalLabels: stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} podLabels: stackable.tech/vendor: Stackable + stackable.tech/stack: {{ stackLabel }} rootUser: admin rootPassword: adminadmin mode: distributed diff --git a/stacks/data-lakehouse-iceberg-trino-spark/s3-connection.yaml b/stacks/data-lakehouse-iceberg-trino-spark/s3-connection.yaml index 411d7a4d..610e6541 100644 --- a/stacks/data-lakehouse-iceberg-trino-spark/s3-connection.yaml +++ b/stacks/data-lakehouse-iceberg-trino-spark/s3-connection.yaml @@ -4,7 +4,7 @@ kind: S3Connection metadata: name: minio spec: - host: minio.default.svc.cluster.local + host: minio.{{ NAMESPACE }}.svc.cluster.local port: 9000 accessStyle: Path credentials: diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index 5ca4ce3b..ddd31c6e 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -231,12 +231,15 @@ stacks: - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/data-lakehouse-iceberg-trino-spark/kafka.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/nifi-kafka-druid-superset-s3/superset.yaml # Reuse - supportedNamespaces: ["default"] # until namespace can be templates, the minio FQDN hard-codes the namespace + supportedNamespaces: [] resourceRequests: cpu: "71" memory: 160Gi pvc: 1Ti parameters: + - name: stackLabel + description: Stack label to add to various components + default: data-lakehouse-iceberg-trino-spark - name: trinoAdminPassword description: Password of the Trino admin user default: adminadmin From 62849d75da63116d327dc73cd098cc7e0f4bd7ba Mon Sep 17 00:00:00 2001 From: xeniape Date: Tue, 10 Mar 2026 20:52:42 +0100 Subject: [PATCH 18/22] remove default namespace warnings in docs --- .../demos/pages/data-lakehouse-iceberg-trino-spark.adoc | 6 ------ .../demos/pages/nifi-kafka-druid-earthquake-data.adoc | 6 ------ .../demos/pages/nifi-kafka-druid-water-level-data.adoc | 6 ------ 3 files changed, 18 deletions(-) diff --git a/docs/modules/demos/pages/data-lakehouse-iceberg-trino-spark.adoc b/docs/modules/demos/pages/data-lakehouse-iceberg-trino-spark.adoc index 96429e96..8c9be204 100644 --- a/docs/modules/demos/pages/data-lakehouse-iceberg-trino-spark.adoc +++ b/docs/modules/demos/pages/data-lakehouse-iceberg-trino-spark.adoc @@ -18,12 +18,6 @@ There is also the smaller xref:trino-iceberg.adoc[] demo focusing on the abiliti The `trino-iceberg` demo has no streaming data and can be executed on a local workstation. ==== -[CAUTION] -==== -This demo only runs in the `default` namespace, as a `ServiceAccount` will be created. -Additionally, we have to use the FQDN service names (including the namespace), so that the used TLS certificates are valid. -==== - Install this demo on an existing Kubernetes cluster: [source,console] diff --git a/docs/modules/demos/pages/nifi-kafka-druid-earthquake-data.adoc b/docs/modules/demos/pages/nifi-kafka-druid-earthquake-data.adoc index feb95649..434a5bf0 100644 --- a/docs/modules/demos/pages/nifi-kafka-druid-earthquake-data.adoc +++ b/docs/modules/demos/pages/nifi-kafka-druid-earthquake-data.adoc @@ -15,12 +15,6 @@ Install this demo on an existing Kubernetes cluster: $ stackablectl demo install nifi-kafka-druid-earthquake-data ---- -[CAUTION] -==== -This demo only runs in the `default` namespace, as a ServiceAccount will be created. -Additionally, we have to use the FQDN service names (including the namespace), so that the used TLS certificates are valid. -==== - [#system-requirements] == System requirements diff --git a/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc b/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc index 7db78cbd..838c6845 100644 --- a/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc +++ b/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc @@ -15,12 +15,6 @@ Install this demo on an existing Kubernetes cluster: $ stackablectl demo install nifi-kafka-druid-water-level-data ---- -[CAUTION] -==== -This demo only runs in the `default` namespace, as a `ServiceAccount` will be created. Additionally, we have to use the -FQDN service names (including the namespace), so that the used TLS certificates are valid. -==== - [#system-requirements] == System requirements From f8b5da08618d23ca4030e16830a34330db9f2496 Mon Sep 17 00:00:00 2001 From: xeniape Date: Wed, 11 Mar 2026 12:03:44 +0100 Subject: [PATCH 19/22] add finalizer explanation --- demos/argo-cd-git-ops/applications/airflow-postgres.yaml | 1 + demos/argo-cd-git-ops/applications/airflow.yaml | 1 + demos/argo-cd-git-ops/applications/minio.yaml | 1 + stacks/argo-cd-git-ops/applications/sealed-secrets.yaml | 1 + 4 files changed, 4 insertions(+) diff --git a/demos/argo-cd-git-ops/applications/airflow-postgres.yaml b/demos/argo-cd-git-ops/applications/airflow-postgres.yaml index 3391bfa8..8de55de9 100644 --- a/demos/argo-cd-git-ops/applications/airflow-postgres.yaml +++ b/demos/argo-cd-git-ops/applications/airflow-postgres.yaml @@ -3,6 +3,7 @@ apiVersion: argoproj.io/v1alpha1 kind: Application metadata: name: airflow-postgres + # Finalizer needed so resources created by the Application are deleted when the Application gets removed finalizers: - resources-finalizer.argocd.argoproj.io spec: diff --git a/demos/argo-cd-git-ops/applications/airflow.yaml b/demos/argo-cd-git-ops/applications/airflow.yaml index 52d4f46f..e0d6df85 100644 --- a/demos/argo-cd-git-ops/applications/airflow.yaml +++ b/demos/argo-cd-git-ops/applications/airflow.yaml @@ -3,6 +3,7 @@ apiVersion: argoproj.io/v1alpha1 kind: Application metadata: name: airflow + # Finalizer needed so resources created by the Application are deleted when the Application gets removed finalizers: - resources-finalizer.argocd.argoproj.io spec: diff --git a/demos/argo-cd-git-ops/applications/minio.yaml b/demos/argo-cd-git-ops/applications/minio.yaml index 61df2292..b7ee2863 100644 --- a/demos/argo-cd-git-ops/applications/minio.yaml +++ b/demos/argo-cd-git-ops/applications/minio.yaml @@ -3,6 +3,7 @@ apiVersion: argoproj.io/v1alpha1 kind: Application metadata: name: minio + # Finalizer needed so resources created by the Application are deleted when the Application gets removed finalizers: - resources-finalizer.argocd.argoproj.io spec: diff --git a/stacks/argo-cd-git-ops/applications/sealed-secrets.yaml b/stacks/argo-cd-git-ops/applications/sealed-secrets.yaml index 350201d1..bf7fb417 100644 --- a/stacks/argo-cd-git-ops/applications/sealed-secrets.yaml +++ b/stacks/argo-cd-git-ops/applications/sealed-secrets.yaml @@ -3,6 +3,7 @@ apiVersion: argoproj.io/v1alpha1 kind: Application metadata: name: sealed-secrets + # Finalizer needed so resources created by the Application are deleted when the Application gets removed finalizers: - resources-finalizer.argocd.argoproj.io spec: From 9f87a91c333ae14b584e5033035c3755b0e12b91 Mon Sep 17 00:00:00 2001 From: xeniape Date: Wed, 11 Mar 2026 12:10:13 +0100 Subject: [PATCH 20/22] parameterize demo label --- demos/demos-v2.yaml | 4 ++++ demos/end-to-end-security/create-spark-report.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/demos/demos-v2.yaml b/demos/demos-v2.yaml index a1d3bc3e..d78d4eb3 100644 --- a/demos/demos-v2.yaml +++ b/demos/demos-v2.yaml @@ -97,6 +97,10 @@ demos: cpu: 9000m memory: 19586Mi pvc: 40Gi + parameters: + - name: demoLabel + description: Demo label to add to various components + default: argo-cd-git-ops nifi-kafka-druid-earthquake-data: description: Demo ingesting earthquake data into Kafka using NiFi, streaming it into Druid and creating a Superset dashboard documentation: https://docs.stackable.tech/home/stable/demos/nifi-kafka-druid-earthquake-data diff --git a/demos/end-to-end-security/create-spark-report.yaml b/demos/end-to-end-security/create-spark-report.yaml index 5c943eb2..d38719fe 100644 --- a/demos/end-to-end-security/create-spark-report.yaml +++ b/demos/end-to-end-security/create-spark-report.yaml @@ -55,7 +55,7 @@ data: name: spark-report labels: stackable.tech/vendor: Stackable - stackable.tech/demo: end-to-end-security + stackable.tech/demo: {{ demoLabel }} spec: sparkImage: productVersion: 4.0.1 From 5dbe361b600ed104000b5a834ec021c93d685d36 Mon Sep 17 00:00:00 2001 From: xeniape Date: Wed, 11 Mar 2026 12:43:23 +0100 Subject: [PATCH 21/22] add/remove namespace in rbac --- demos/end-to-end-security/serviceaccount.yaml | 1 + stacks/_templates/keycloak-serviceaccount.yaml | 1 + stacks/signal-processing/serviceaccount.yaml | 1 - 3 files changed, 2 insertions(+), 1 deletion(-) diff --git a/demos/end-to-end-security/serviceaccount.yaml b/demos/end-to-end-security/serviceaccount.yaml index df7d2cf0..dd1c8e36 100644 --- a/demos/end-to-end-security/serviceaccount.yaml +++ b/demos/end-to-end-security/serviceaccount.yaml @@ -11,6 +11,7 @@ metadata: subjects: - kind: ServiceAccount name: demo-serviceaccount + namespace: {{ NAMESPACE }} roleRef: kind: ClusterRole name: demo-clusterrole diff --git a/stacks/_templates/keycloak-serviceaccount.yaml b/stacks/_templates/keycloak-serviceaccount.yaml index 16ac31d8..99075e7d 100644 --- a/stacks/_templates/keycloak-serviceaccount.yaml +++ b/stacks/_templates/keycloak-serviceaccount.yaml @@ -11,6 +11,7 @@ metadata: subjects: - kind: ServiceAccount name: keycloak-serviceaccount + namespace: {{ NAMESPACE }} roleRef: kind: ClusterRole name: keycloak-clusterrole diff --git a/stacks/signal-processing/serviceaccount.yaml b/stacks/signal-processing/serviceaccount.yaml index d1e4901e..f8298b4b 100644 --- a/stacks/signal-processing/serviceaccount.yaml +++ b/stacks/signal-processing/serviceaccount.yaml @@ -11,7 +11,6 @@ metadata: subjects: - kind: ServiceAccount name: spark - namespace: {{ NAMESPACE }} roleRef: kind: Role name: spark-role From decf8746eb51e1d3ef15dbf1b07629ef3ddc51ff Mon Sep 17 00:00:00 2001 From: xeniape Date: Thu, 12 Mar 2026 13:37:32 +0100 Subject: [PATCH 22/22] use DEMO and STACK parameter for labels etc. --- .../sealed-airflow-postgres-credentials.yaml | 2 +- .../manifests/airflow/airflow.yaml | 2 +- .../airflow/sealed-airflow-credentials.yaml | 2 +- .../sealed-airflow-minio-connection.yaml | 2 +- .../manifests/minio/minio.yaml | 16 +++---- .../minio/sealed-minio-credentials.yaml | 2 +- .../create-spark-ingestion-job.yaml | 2 +- demos/demos-v2.yaml | 13 +----- .../create-spark-report.yaml | 2 +- stacks/_templates/argo-cd.yaml | 2 +- stacks/_templates/keycloak.yaml | 4 +- .../minio-distributed-tls/values.yaml | 4 +- stacks/_templates/minio.yaml | 4 +- stacks/_templates/opensearch-dashboards.yaml | 4 +- stacks/_templates/postgresql-airflow.yaml | 2 +- stacks/_templates/postgresql-druid.yaml | 2 +- .../_templates/postgresql-hive-iceberg.yaml | 2 +- stacks/_templates/postgresql-hive.yaml | 2 +- stacks/_templates/postgresql-superset.yaml | 2 +- stacks/_templates/postgresql-timescaledb.yaml | 2 +- stacks/_templates/vector-aggregator.yaml | 4 +- .../secrets/sealed-secrets-key.yaml | 2 +- stacks/end-to-end-security/krb5.yaml | 2 +- .../jupyterhub-native-auth.yaml | 2 +- stacks/jupyterhub-keycloak/jupyterhub.yaml | 2 +- stacks/jupyterhub-keycloak/keycloak.yaml | 4 +- stacks/monitoring/prometheus.yaml | 2 +- stacks/observability/grafana-tempo.yaml | 2 +- stacks/observability/grafana.yaml | 2 +- stacks/observability/jaeger.yaml | 2 +- .../observability/opentelemetry-operator.yaml | 2 +- stacks/opensearch-rag/jupyterlab.yaml | 4 +- stacks/signal-processing/grafana.yaml | 2 +- stacks/signal-processing/jupyterhub.yaml | 4 +- stacks/stacks-v2.yaml | 42 ------------------- 35 files changed, 48 insertions(+), 103 deletions(-) diff --git a/demos/argo-cd-git-ops/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml b/demos/argo-cd-git-ops/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml index 7e69fd60..96dcd8f0 100644 --- a/demos/argo-cd-git-ops/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml +++ b/demos/argo-cd-git-ops/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml @@ -18,5 +18,5 @@ spec: namespace: stackable-airflow labels: stackable.tech/vendor: Stackable - stackable.tech/demo: {{ demoLabel }} + stackable.tech/demo: {{ DEMO }} type: Opaque diff --git a/demos/argo-cd-git-ops/manifests/airflow/airflow.yaml b/demos/argo-cd-git-ops/manifests/airflow/airflow.yaml index 69aa26fa..490ce6aa 100644 --- a/demos/argo-cd-git-ops/manifests/airflow/airflow.yaml +++ b/demos/argo-cd-git-ops/manifests/airflow/airflow.yaml @@ -5,7 +5,7 @@ metadata: name: airflow labels: stackable.tech/vendor: Stackable - stackable.tech/demo: {{ demoLabel }} + stackable.tech/demo: {{ DEMO }} spec: image: productVersion: 3.1.6 diff --git a/demos/argo-cd-git-ops/manifests/airflow/sealed-airflow-credentials.yaml b/demos/argo-cd-git-ops/manifests/airflow/sealed-airflow-credentials.yaml index b5459962..a0530541 100644 --- a/demos/argo-cd-git-ops/manifests/airflow/sealed-airflow-credentials.yaml +++ b/demos/argo-cd-git-ops/manifests/airflow/sealed-airflow-credentials.yaml @@ -23,5 +23,5 @@ spec: namespace: stackable-airflow labels: stackable.tech/vendor: Stackable - stackable.tech/demo: {{ demoLabel }} + stackable.tech/demo: {{ DEMO }} type: Opaque diff --git a/demos/argo-cd-git-ops/manifests/airflow/sealed-airflow-minio-connection.yaml b/demos/argo-cd-git-ops/manifests/airflow/sealed-airflow-minio-connection.yaml index fb395591..e7dccbac 100644 --- a/demos/argo-cd-git-ops/manifests/airflow/sealed-airflow-minio-connection.yaml +++ b/demos/argo-cd-git-ops/manifests/airflow/sealed-airflow-minio-connection.yaml @@ -15,5 +15,5 @@ spec: namespace: stackable-airflow labels: stackable.tech/vendor: Stackable - stackable.tech/demo: {{ demoLabel }} + stackable.tech/demo: {{ DEMO }} type: Opaque diff --git a/demos/argo-cd-git-ops/manifests/minio/minio.yaml b/demos/argo-cd-git-ops/manifests/minio/minio.yaml index 50ad9dd9..7f2c5217 100644 --- a/demos/argo-cd-git-ops/manifests/minio/minio.yaml +++ b/demos/argo-cd-git-ops/manifests/minio/minio.yaml @@ -6,7 +6,7 @@ metadata: name: "minio-sa" labels: stackable.tech/vendor: Stackable - stackable.tech/demo: {{ demoLabel }} + stackable.tech/demo: {{ DEMO }} --- # Source: minio/templates/configmap.yaml apiVersion: v1 @@ -19,7 +19,7 @@ metadata: release: minio heritage: Helm stackable.tech/vendor: Stackable - stackable.tech/demo: {{ demoLabel }} + stackable.tech/demo: {{ DEMO }} data: initialize: |- #!/bin/sh @@ -414,7 +414,7 @@ metadata: release: minio heritage: Helm stackable.tech/vendor: Stackable - stackable.tech/demo: {{ demoLabel }} + stackable.tech/demo: {{ DEMO }} spec: accessModes: - "ReadWriteOnce" @@ -433,7 +433,7 @@ metadata: release: minio heritage: Helm stackable.tech/vendor: Stackable - stackable.tech/demo: {{ demoLabel }} + stackable.tech/demo: {{ DEMO }} spec: type: NodePort externalTrafficPolicy: "Cluster" @@ -458,7 +458,7 @@ metadata: heritage: Helm monitoring: "true" stackable.tech/vendor: Stackable - stackable.tech/demo: {{ demoLabel }} + stackable.tech/demo: {{ DEMO }} spec: type: NodePort externalTrafficPolicy: "Cluster" @@ -482,7 +482,7 @@ metadata: release: minio heritage: Helm stackable.tech/vendor: Stackable - stackable.tech/demo: {{ demoLabel }} + stackable.tech/demo: {{ DEMO }} spec: strategy: type: RollingUpdate @@ -501,7 +501,7 @@ spec: app: minio release: minio stackable.tech/vendor: Stackable - stackable.tech/demo: {{ demoLabel }} + stackable.tech/demo: {{ DEMO }} annotations: checksum/secrets: fa63e34a92c817c84057e2d452fa683e66462a57b0529388fb96a57e05f38e57 checksum/config: ebea49cc4c1bfbd1b156a58bf770a776ff87fe199f642d31c2816b5515112e72 @@ -597,7 +597,7 @@ metadata: release: minio heritage: Helm stackable.tech/vendor: Stackable - stackable.tech/demo: {{ demoLabel }} + stackable.tech/demo: {{ DEMO }} annotations: "helm.sh/hook": post-install,post-upgrade "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation diff --git a/demos/argo-cd-git-ops/manifests/minio/sealed-minio-credentials.yaml b/demos/argo-cd-git-ops/manifests/minio/sealed-minio-credentials.yaml index 022eb7d5..94f672e4 100644 --- a/demos/argo-cd-git-ops/manifests/minio/sealed-minio-credentials.yaml +++ b/demos/argo-cd-git-ops/manifests/minio/sealed-minio-credentials.yaml @@ -16,5 +16,5 @@ spec: namespace: minio labels: stackable.tech/vendor: Stackable - stackable.tech/demo: {{ demoLabel }} + stackable.tech/demo: {{ DEMO }} type: Opaque diff --git a/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml b/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml index 3e6e8408..0e11e858 100644 --- a/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml +++ b/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml @@ -142,7 +142,7 @@ data: name: spark-ingest-into-lakehouse labels: stackable.tech/vendor: Stackable - stackable.tech/demo: {{ demoLabel }} + stackable.tech/demo: {{ DEMO }} spec: sparkImage: productVersion: 4.0.1 diff --git a/demos/demos-v2.yaml b/demos/demos-v2.yaml index d78d4eb3..5360da21 100644 --- a/demos/demos-v2.yaml +++ b/demos/demos-v2.yaml @@ -33,9 +33,6 @@ demos: memory: 15Gi pvc: 15Gi parameters: - - name: demoLabel - description: Demo label to add to various components - default: argo-cd-git-ops - name: customGitUrl description: The URL to the Git repository (this can be adapted in case of forking the repository) to demonstrate syncing via Argo and Git. default: https://github.com/stackabletech/demos/ @@ -97,10 +94,7 @@ demos: cpu: 9000m memory: 19586Mi pvc: 40Gi - parameters: - - name: demoLabel - description: Demo label to add to various components - default: argo-cd-git-ops + parameters: [] nifi-kafka-druid-earthquake-data: description: Demo ingesting earthquake data into Kafka using NiFi, streaming it into Druid and creating a Superset dashboard documentation: https://docs.stackable.tech/home/stable/demos/nifi-kafka-druid-earthquake-data @@ -227,10 +221,7 @@ demos: cpu: "80" memory: 200Gi pvc: 300Gi - parameters: - - name: demoLabel - description: Demo label to add to various components - default: data-lakehouse-iceberg-trino-spark + parameters: [] jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data: description: Jupyterhub with PySpark and HDFS integration documentation: https://docs.stackable.tech/home/stable/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data diff --git a/demos/end-to-end-security/create-spark-report.yaml b/demos/end-to-end-security/create-spark-report.yaml index d38719fe..f4f77038 100644 --- a/demos/end-to-end-security/create-spark-report.yaml +++ b/demos/end-to-end-security/create-spark-report.yaml @@ -55,7 +55,7 @@ data: name: spark-report labels: stackable.tech/vendor: Stackable - stackable.tech/demo: {{ demoLabel }} + stackable.tech/demo: {{ DEMO }} spec: sparkImage: productVersion: 4.0.1 diff --git a/stacks/_templates/argo-cd.yaml b/stacks/_templates/argo-cd.yaml index 7590d9ac..5f72f8a6 100644 --- a/stacks/_templates/argo-cd.yaml +++ b/stacks/_templates/argo-cd.yaml @@ -9,7 +9,7 @@ options: global: additionalLabels: stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} configs: secret: argocdServerAdminPassword: "{{ argocdAdminPassword }}" diff --git a/stacks/_templates/keycloak.yaml b/stacks/_templates/keycloak.yaml index 2983f5ab..d9bfe4d4 100644 --- a/stacks/_templates/keycloak.yaml +++ b/stacks/_templates/keycloak.yaml @@ -66,8 +66,6 @@ spec: KEYCLOAK_HOSTNAME="$(echo $KEYCLOAK_ADDRESS | grep -oP '^[^:]+')" KEYCLOAK_PORT="$(echo $KEYCLOAK_ADDRESS | grep -oP '[0-9]+$')" - STACK_LABEL=$(kubectl get svc keycloak -n {{ NAMESPACE }} -o json | jq '.metadata.labels."stackable.tech/stack"') - cat << EOF | kubectl apply -f - apiVersion: authentication.stackable.tech/v1alpha1 kind: AuthenticationClass @@ -75,7 +73,7 @@ spec: name: keycloak labels: stackable.tech/vendor: Stackable - stackable.tech/stack: $STACK_LABEL + stackable.tech/stack: {{ STACK }} spec: provider: oidc: diff --git a/stacks/_templates/minio-distributed-tls/values.yaml b/stacks/_templates/minio-distributed-tls/values.yaml index 570e9748..653d269c 100644 --- a/stacks/_templates/minio-distributed-tls/values.yaml +++ b/stacks/_templates/minio-distributed-tls/values.yaml @@ -1,9 +1,9 @@ additionalLabels: stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} podLabels: stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} rootUser: admin rootPassword: adminadmin mode: distributed diff --git a/stacks/_templates/minio.yaml b/stacks/_templates/minio.yaml index 18147a4c..d5e0440c 100644 --- a/stacks/_templates/minio.yaml +++ b/stacks/_templates/minio.yaml @@ -8,10 +8,10 @@ version: 5.4.0 # RELEASE.2024-12-18T13-15-44Z options: additionalLabels: stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} podLabels: stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} rootUser: admin rootPassword: {{ minioAdminPassword }} mode: standalone diff --git a/stacks/_templates/opensearch-dashboards.yaml b/stacks/_templates/opensearch-dashboards.yaml index 5ffcb29e..721b3afa 100644 --- a/stacks/_templates/opensearch-dashboards.yaml +++ b/stacks/_templates/opensearch-dashboards.yaml @@ -11,7 +11,7 @@ options: tag: "{{ opensearchVersion }}-stackable{{ stackableReleaseVersion }}" labels: stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} service: type: NodePort port: 5601 @@ -20,7 +20,7 @@ options: stackable.tech/logging-credentials-secret: opensearch-user labels: stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} opensearchHosts: null # Use the discovery ConfigMap instead extraEnvs: - name: OPENSEARCH_HOSTS diff --git a/stacks/_templates/postgresql-airflow.yaml b/stacks/_templates/postgresql-airflow.yaml index dda36a74..b5bc445a 100644 --- a/stacks/_templates/postgresql-airflow.yaml +++ b/stacks/_templates/postgresql-airflow.yaml @@ -8,7 +8,7 @@ version: 18.1.3 # 18.0.0 options: commonLabels: stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} auth: username: airflow password: airflow diff --git a/stacks/_templates/postgresql-druid.yaml b/stacks/_templates/postgresql-druid.yaml index 96ae74a3..66ba2d31 100644 --- a/stacks/_templates/postgresql-druid.yaml +++ b/stacks/_templates/postgresql-druid.yaml @@ -8,7 +8,7 @@ version: 18.1.3 # 18.0.0 options: commonLabels: stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} auth: username: druid password: druid diff --git a/stacks/_templates/postgresql-hive-iceberg.yaml b/stacks/_templates/postgresql-hive-iceberg.yaml index ba9ddbd3..cd3cf64f 100644 --- a/stacks/_templates/postgresql-hive-iceberg.yaml +++ b/stacks/_templates/postgresql-hive-iceberg.yaml @@ -8,7 +8,7 @@ version: 18.1.3 # 18.0.0 options: commonLabels: stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} auth: username: hive password: hive diff --git a/stacks/_templates/postgresql-hive.yaml b/stacks/_templates/postgresql-hive.yaml index 12693c37..64597889 100644 --- a/stacks/_templates/postgresql-hive.yaml +++ b/stacks/_templates/postgresql-hive.yaml @@ -8,7 +8,7 @@ version: 18.1.3 # 18.0.0 options: commonLabels: stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} auth: username: hive password: hive diff --git a/stacks/_templates/postgresql-superset.yaml b/stacks/_templates/postgresql-superset.yaml index cbb3af09..162871ee 100644 --- a/stacks/_templates/postgresql-superset.yaml +++ b/stacks/_templates/postgresql-superset.yaml @@ -8,7 +8,7 @@ version: 18.1.3 # 18.0.0 options: commonLabels: stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} auth: username: superset password: superset diff --git a/stacks/_templates/postgresql-timescaledb.yaml b/stacks/_templates/postgresql-timescaledb.yaml index 61611a90..78b7aeb4 100644 --- a/stacks/_templates/postgresql-timescaledb.yaml +++ b/stacks/_templates/postgresql-timescaledb.yaml @@ -8,7 +8,7 @@ version: 0.33.1 options: podLabels: stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} replicaCount: 1 secrets: credentials: diff --git a/stacks/_templates/vector-aggregator.yaml b/stacks/_templates/vector-aggregator.yaml index 80593204..974a5371 100644 --- a/stacks/_templates/vector-aggregator.yaml +++ b/stacks/_templates/vector-aggregator.yaml @@ -8,10 +8,10 @@ version: 0.49.0 # app version 0.52.0 options: commonLabels: stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} podLabels: stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} role: Aggregator customConfig: sources: diff --git a/stacks/argo-cd-git-ops/secrets/sealed-secrets-key.yaml b/stacks/argo-cd-git-ops/secrets/sealed-secrets-key.yaml index bc196947..cb25cf58 100644 --- a/stacks/argo-cd-git-ops/secrets/sealed-secrets-key.yaml +++ b/stacks/argo-cd-git-ops/secrets/sealed-secrets-key.yaml @@ -5,7 +5,7 @@ metadata: name: sealed-secrets-key labels: stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} stringData: # Generated on 2025/04/12 # openssl req -x509 -nodes -newkey rsa:4096 -keyout tls.key -out tls.crt -subj "/CN=sealed-secrets" -days 3650 diff --git a/stacks/end-to-end-security/krb5.yaml b/stacks/end-to-end-security/krb5.yaml index bb45e4e6..6bd2ca63 100644 --- a/stacks/end-to-end-security/krb5.yaml +++ b/stacks/end-to-end-security/krb5.yaml @@ -79,7 +79,7 @@ spec: name: data labels: stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} spec: accessModes: - ReadWriteOnce diff --git a/stacks/jupyterhub-keycloak/jupyterhub-native-auth.yaml b/stacks/jupyterhub-keycloak/jupyterhub-native-auth.yaml index 41b44527..19e404a3 100644 --- a/stacks/jupyterhub-keycloak/jupyterhub-native-auth.yaml +++ b/stacks/jupyterhub-keycloak/jupyterhub-native-auth.yaml @@ -40,7 +40,7 @@ options: enabled: false extraLabels: stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} profileList: - display_name: "Default" description: "Default profile" diff --git a/stacks/jupyterhub-keycloak/jupyterhub.yaml b/stacks/jupyterhub-keycloak/jupyterhub.yaml index 2e6a0e6a..903451cc 100644 --- a/stacks/jupyterhub-keycloak/jupyterhub.yaml +++ b/stacks/jupyterhub-keycloak/jupyterhub.yaml @@ -149,7 +149,7 @@ options: enabled: false extraLabels: stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} initContainers: - name: download-notebook image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev diff --git a/stacks/jupyterhub-keycloak/keycloak.yaml b/stacks/jupyterhub-keycloak/keycloak.yaml index 0069d741..59c54b31 100644 --- a/stacks/jupyterhub-keycloak/keycloak.yaml +++ b/stacks/jupyterhub-keycloak/keycloak.yaml @@ -68,8 +68,6 @@ spec: KEYCLOAK_HOSTNAME="$(echo $KEYCLOAK_ADDRESS | grep -oP '^[^:]+')" KEYCLOAK_PORT="$(echo $KEYCLOAK_ADDRESS | grep -oP '[0-9]+$')" - STACK_LABEL=$(kubectl get svc keycloak -n {{ NAMESPACE }} -o json | jq '.metadata.labels."stackable.tech/stack"') - cat << EOF | kubectl apply -f - apiVersion: v1 kind: ConfigMap @@ -77,7 +75,7 @@ spec: name: keycloak-address labels: stackable.tech/vendor: Stackable - stackable.tech/stack: $STACK_LABEL + stackable.tech/stack: {{ STACK }} data: keycloakAddress: "$KEYCLOAK_HOSTNAME:$KEYCLOAK_PORT" keycloakNodeIp: "$KEYCLOAK_HOSTNAME" diff --git a/stacks/monitoring/prometheus.yaml b/stacks/monitoring/prometheus.yaml index 23f7577b..1f796118 100644 --- a/stacks/monitoring/prometheus.yaml +++ b/stacks/monitoring/prometheus.yaml @@ -8,7 +8,7 @@ version: 77.12.0 # 0.85.0 options: commonLabels: stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} prometheusOperator: kubeletService: namespace: {{ NAMESPACE }} diff --git a/stacks/observability/grafana-tempo.yaml b/stacks/observability/grafana-tempo.yaml index d6f599be..3dee987b 100644 --- a/stacks/observability/grafana-tempo.yaml +++ b/stacks/observability/grafana-tempo.yaml @@ -15,7 +15,7 @@ options: global: commonLabels: stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} tempo: receivers: otlp: diff --git a/stacks/observability/grafana.yaml b/stacks/observability/grafana.yaml index 8319b97c..a80f6e68 100644 --- a/stacks/observability/grafana.yaml +++ b/stacks/observability/grafana.yaml @@ -13,7 +13,7 @@ version: 10.1.4 # 12.2.1 options: extraLabels: stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} admin: existingSecret: grafana-admin-credentials service: diff --git a/stacks/observability/jaeger.yaml b/stacks/observability/jaeger.yaml index fa99b017..ce36aea3 100644 --- a/stacks/observability/jaeger.yaml +++ b/stacks/observability/jaeger.yaml @@ -13,7 +13,7 @@ version: 3.4.1 # 1.53.0 options: commonLabels: stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} provisionDataStore: cassandra: false allInOne: diff --git a/stacks/observability/opentelemetry-operator.yaml b/stacks/observability/opentelemetry-operator.yaml index c93debf5..360abf07 100644 --- a/stacks/observability/opentelemetry-operator.yaml +++ b/stacks/observability/opentelemetry-operator.yaml @@ -10,7 +10,7 @@ version: 0.99.0 # 0.138.0 options: additionalLabels: stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} admissionWebhooks: certManager: enabled: false diff --git a/stacks/opensearch-rag/jupyterlab.yaml b/stacks/opensearch-rag/jupyterlab.yaml index 559c60a7..76276802 100644 --- a/stacks/opensearch-rag/jupyterlab.yaml +++ b/stacks/opensearch-rag/jupyterlab.yaml @@ -12,13 +12,13 @@ spec: matchLabels: app: jupyterlab stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} template: metadata: labels: app: jupyterlab stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} spec: serviceAccountName: default containers: diff --git a/stacks/signal-processing/grafana.yaml b/stacks/signal-processing/grafana.yaml index 85b4a6d2..8fd26f04 100644 --- a/stacks/signal-processing/grafana.yaml +++ b/stacks/signal-processing/grafana.yaml @@ -8,7 +8,7 @@ version: 10.1.2 # 12.2.0 options: extraLabels: stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} grafana.ini: analytics: check_for_updates: false diff --git a/stacks/signal-processing/jupyterhub.yaml b/stacks/signal-processing/jupyterhub.yaml index dfced547..7f5d2536 100644 --- a/stacks/signal-processing/jupyterhub.yaml +++ b/stacks/signal-processing/jupyterhub.yaml @@ -17,7 +17,7 @@ options: authenticator_class: dummy labels: stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} proxy: service: type: NodePort @@ -40,7 +40,7 @@ options: enabled: false extraLabels: stackable.tech/vendor: Stackable - stackable.tech/stack: {{ stackLabel }} + stackable.tech/stack: {{ STACK }} initContainers: - name: download-notebook image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index ce361e0e..96b0c840 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -33,9 +33,6 @@ stacks: cpu: 5000m pvc: 5Gi parameters: - - name: stackLabel - description: Stack label to add to various components - default: argo-cd-git-ops - name: stackableReleaseVersion description: The Stackable release to be installed via Argo. This replaces the demo `stackableRelease` parameter. # TODO(@maltesander): Must be adapted in case of release ! @@ -67,9 +64,6 @@ stacks: memory: 1714Mi pvc: 110Gi parameters: - - name: stackLabel - description: Stack label to add to various components - default: monitoring - name: grafanaAdminPassword description: Password of the Grafana admin user default: adminadmin @@ -99,9 +93,6 @@ stacks: memory: 3147Mi pvc: 24Gi parameters: - - name: stackLabel - description: Stack label to add to various components - default: logging - name: openSearchAdminPassword description: Password of the OpenSearch admin user default: adminadmin @@ -137,9 +128,6 @@ stacks: - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/observability/opentelemetry-collector-sidecar.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/observability/opentelemetry-collector-deployment.yaml parameters: - - name: stackLabel - description: Stack label to add to various components - default: observability - name: grafanaAdminPassword description: Password of the Grafana admin user default: adminadmin @@ -181,9 +169,6 @@ stacks: memory: 9010Mi pvc: 24Gi parameters: - - name: stackLabel - description: Stack label to add to various components - default: airflow - name: trinoAdminPassword description: Password of the Trino admin user default: adminadmin @@ -237,9 +222,6 @@ stacks: memory: 160Gi pvc: 1Ti parameters: - - name: stackLabel - description: Stack label to add to various components - default: data-lakehouse-iceberg-trino-spark - name: trinoAdminPassword description: Password of the Trino admin user default: adminadmin @@ -312,9 +294,6 @@ stacks: memory: 42330Mi pvc: 75Gi parameters: - - name: stackLabel - description: Stack label to add to various components - default: nifi-kafka-druid-superset-s3 - name: nifiAdminPassword description: Password of the NiFI admin user default: adminadmin @@ -360,9 +339,6 @@ stacks: memory: 15910Mi pvc: 36Gi parameters: - - name: stackLabel - description: Stack label to add to various components - default: spark-trino-superset-s3 - name: trinoAdminPassword description: Password of the Trino admin user default: adminadmin @@ -405,9 +381,6 @@ stacks: memory: 15822Mi pvc: 28Gi parameters: - - name: stackLabel - description: Stack label to add to various components - default: trino-superset-s3 - name: trinoAdminPassword description: Password of the Trino admin user default: adminadmin @@ -448,9 +421,6 @@ stacks: memory: 21Gi pvc: 110Gi # 100Gi for MinIO parameters: - - name: stackLabel - description: Stack label to add to various components - default: trino-iceberg - name: trinoAdminPassword description: Password of the Trino admin user default: adminadmin @@ -585,9 +555,6 @@ stacks: - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/setup-postgresql.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/superset.yaml parameters: - - name: stackLabel - description: Stack label to add to various components - default: end-to-end-security - name: keycloakAdminPassword description: Password of the Keycloak admin user default: adminadmin @@ -636,9 +603,6 @@ stacks: - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/signal-processing/spark_driver_service.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/signal-processing/jupyterhub.yaml parameters: - - name: stackLabel - description: Stack label to add to various components - default: signal-processing - name: nifiAdminPassword description: Password of the NiFI admin user default: adminadmin @@ -677,9 +641,6 @@ stacks: - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/jupyterhub-keycloak/serviceaccount.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/jupyterhub-keycloak/s3-connection.yaml parameters: - - name: stackLabel - description: Stack label to add to various components - default: jupyterhub-keycloak - name: keycloakAdminPassword description: Password of the Keycloak admin user default: adminadmin @@ -714,9 +675,6 @@ stacks: memory: 16Gi pvc: 10Gi parameters: - - name: stackLabel - description: Stack label to add to various components - default: opensearch-rag - name: openSearchAdminPassword description: Password of the OpenSearch admin user default: adminadmin