Skip to content
Snippets Groups Projects
Unverified Commit 7d88f18b authored by v-yussupov's avatar v-yussupov Committed by GitHub
Browse files

Merge pull request #105 from jakovitssecproj/data_pipeline_confitional_features 

Data pipeline confitional features
parents ed65ef7e 938ca68a
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 930 additions and 0 deletions
nodetypes/radon.nodes.datapipeline.process/attrMatchCondition/NodeType.tosca 0 → 100644
+ 78
− 0
View file @ 7d88f18b
tosca_definitions_version: tosca_simple_yaml_1_3
node_types:
radon.nodes.datapipeline.process.attrMatchCondition:
derived_from: radon.nodes.datapipeline.process.LocalAction
metadata:
targetNamespace: "radon.nodes.datapipeline.process"
abstract: "false"
final: "false"
attributes:
template_name:
type: string
default: "attrmatchtemp"
properties:
matchvalue:
type: string
description: Value to compare to
required: false
default: 1000
matchcondition:
type: string
description: Comparison operator, supported, gt,equals,lt
default: "gt"
matchkey:
type: string
description: Name of the attribute (fileSize, )
default: "fileSize"
interfaces:
Standard:
type: tosca.interfaces.node.lifecycle.Standard
operations:
create:
description: The standard create operation
inputs:
template_name:
type: string
required: true
default: { get_attribute: [ SELF, template_name ] }
template_file:
type: string
required: true
default: "attrmatchtemp_localConn.xml"
implementation:
primary: create
dependencies: [ templateFile_localConn ]
timeout: 0
configure:
description: The standard configure operation
inputs:
matchcondition:
type: string
required: true
default: { get_property: [ SELF, matchcondition ] }
matchvalue:
type: string
required: true
default: { get_property: [ SELF, matchvalue ] }
pipeline_id:
type: string
required: true
default: { get_attribute: [ SELF, id ] }
matchkey:
type: string
required: true
default: { get_property: [ SELF, matchkey ] }
implementation:
primary: configure
timeout: 0
artifacts:
templateFile_localConn:
type: radon.artifacts.Ansible
file: attrmatchtemp_localConn.xml
create:
type: radon.artifacts.Ansible
file: create.yml
configure:
type: radon.artifacts.Ansible
file: configure.yml
nodetypes/radon.nodes.datapipeline.process/attrMatchCondition/appearance/bigIcon.png 0 → 100644
NaN GiB (NaN%)
View file @ 7d88f18b
File suppressed by a .gitattributes entry or the file's encoding is unsupported.
nodetypes/radon.nodes.datapipeline.process/attrMatchCondition/appearance/bigIcon.png.mimetype 0 → 100644
+ 1
− 0
View file @ 7d88f18b
image/png
\ No newline at end of file
nodetypes/radon.nodes.datapipeline.process/attrMatchCondition/files/configure/configure.yml 0 → 100644
+ 32
− 0
View file @ 7d88f18b
---
- hosts: all
vars:
get_id1: "processors[0].id"
port: "{{ (lookup('file', '/tmp/nifi.json') | from_json) | default('8080', true) }}"
condition: "{{matchkey}}:{{matchcondition}}({{matchvalue}})"
tasks:
- name: Get list of available NiFi processors within the process group
uri:
url: "http://localhost:{{ port }}/nifi-api/process-groups/{{ pipeline_id }}/processors"
method: Get
register: nifi_resources
- name: Configure the first processor within the process group
vars:
first_prcsr_id: "{{ nifi_resources.json | to_json | from_json | json_query(get_id1) }}"
schedulingPeriodValue: "{{ '0 sec' if schedulingStrategy == 'EVENT_DRIVEN' else schedulingPeriodCRON }}"
schedulingStrategyValue: "{{ 'TIMER_DRIVEN' if schedulingStrategy == 'EVENT_DRIVEN' else schedulingStrategy }}"
uri:
url: "http://localhost:{{ port }}/nifi-api/processors/{{ first_prcsr_id }}"
method: PUT
body:
"revision": {"version": 0}
"component":
"id": "{{ first_prcsr_id }}"
"config":
"properties": {
"conditionmatched":"${ {{condition}} }"
}
"schedulingStrategy": "{{ schedulingStrategyValue }}"
"schedulingPeriod": "{{ schedulingPeriodValue }}"
status_code: 200
body_format: json
nodetypes/radon.nodes.datapipeline.process/attrMatchCondition/files/configure/configure.yml.mimetype 0 → 100644
+ 1
− 0
View file @ 7d88f18b
text/plain
\ No newline at end of file
nodetypes/radon.nodes.datapipeline.process/attrMatchCondition/files/create/create.yml 0 → 100644
+ 44
− 0
View file @ 7d88f18b
---
- hosts: all
vars:
query: "resources[?name=='{{ template_name }}' && starts_with(identifier, '/templates/')].identifier"
port: "{{ (lookup('file', '/tmp/nifi.json') | from_json) | default('8080', true) }}"
tasks:
- name: Copy template file to remote system (shell command requires file to be in remote system)
copy:
src: "{{ template_file }}"
dest: template.xml
- name: Wait for port 8080 to become open on the host ( dont start checking for 4 seconds )
wait_for:
port: "{{ port }}"
timeout: 180
- name: upload template file to NiFi template repository
shell: 'curl -X "POST" "http://localhost:{{ port }}/nifi-api/process-groups/root/templates/upload" \
-H "Content-Type: multipart/form-data" \
-k \
--form template=@template.xml'
register: response
- name: Get list of available NiFi resources for fetching the id of the template by its name
uri:
url: http://localhost:{{ port }}/nifi-api/resources
method: Get
register: nifi_resources
- name: Initiate the template
vars:
template_id: "{{ nifi_resources.json | to_json | from_json | json_query(query) | first }}"
uri:
url: http://localhost:{{ port }}/nifi-api/process-groups/root/template-instance
method: POST
body:
"originX": 2.0
"originY": 3.0
"templateId": "{{ template_id.split('/')[2] }}"
status_code: 201
body_format: json
register: pipeline_info
- name: Set pipeline id property value (process-groups)
set_stats:
data:
id: "{{ pipeline_info.json.flow.processGroups[0].id }}"
# pipeline_type: "processGroups"
when: pipeline_info.json.flow.processGroups|length > 0
nodetypes/radon.nodes.datapipeline.process/attrMatchCondition/files/create/create.yml.mimetype 0 → 100644
+ 1
− 0
View file @ 7d88f18b
text/plain
\ No newline at end of file
nodetypes/radon.nodes.datapipeline.process/attrMatchCondition/files/templateFile_localConn/attrmatchtemp_localConn.xml 0 → 100644
+ 163
− 0
View file @ 7d88f18b
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<template encoding-version="1.3">
<description></description>
<groupId>eb3aee6c-0179-1000-acd4-0e00ef9b3344</groupId>
<name>attrmatchtemp</name>
<snippet>
<processGroups>
<id>82cb54fa-de00-38e0-0000-000000000000</id>
<parentGroupId>b7905d91-01ef-3484-0000-000000000000</parentGroupId>
<position>
<x>0.0</x>
<y>0.0</y>
</position>
<comments></comments>
<contents>
<connections>
<id>356dd247-4795-30b5-0000-000000000000</id>
<parentGroupId>82cb54fa-de00-38e0-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>82cb54fa-de00-38e0-0000-000000000000</groupId>
<id>dc1b668c-8367-3bed-0000-000000000000</id>
<type>OUTPUT_PORT</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
<loadBalancePartitionAttribute></loadBalancePartitionAttribute>
<loadBalanceStatus>LOAD_BALANCE_NOT_CONFIGURED</loadBalanceStatus>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<name></name>
<selectedRelationships>conditionmatched</selectedRelationships>
<source>
<groupId>82cb54fa-de00-38e0-0000-000000000000</groupId>
<id>9b696a42-d43a-304e-0000-000000000000</id>
<type>PROCESSOR</type>
</source>
<zIndex>0</zIndex>
</connections>
<connections>
<id>a9c0614c-e24b-3a92-0000-000000000000</id>
<parentGroupId>82cb54fa-de00-38e0-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>82cb54fa-de00-38e0-0000-000000000000</groupId>
<id>9b696a42-d43a-304e-0000-000000000000</id>
<type>PROCESSOR</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
<loadBalancePartitionAttribute></loadBalancePartitionAttribute>
<loadBalanceStatus>LOAD_BALANCE_NOT_CONFIGURED</loadBalanceStatus>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<name></name>
<source>
<groupId>82cb54fa-de00-38e0-0000-000000000000</groupId>
<id>b24654c3-fbd1-3fa7-0000-000000000000</id>
<type>INPUT_PORT</type>
</source>
<zIndex>0</zIndex>
</connections>
<inputPorts>
<id>b24654c3-fbd1-3fa7-0000-000000000000</id>
<parentGroupId>82cb54fa-de00-38e0-0000-000000000000</parentGroupId>
<position>
<x>216.0</x>
<y>-8.0</y>
</position>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<name>attr_IP_local</name>
<state>STOPPED</state>
<type>INPUT_PORT</type>
<validationErrors>'Port 'attr_IP_local'' is invalid because Port has no incoming connections</validationErrors>
</inputPorts>
<outputPorts>
<id>dc1b668c-8367-3bed-0000-000000000000</id>
<parentGroupId>82cb54fa-de00-38e0-0000-000000000000</parentGroupId>
<position>
<x>216.0</x>
<y>456.0</y>
</position>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<name>attr_OP_local</name>
<state>STOPPED</state>
<type>OUTPUT_PORT</type>
<validationErrors>'Port 'attr_OP_local'' is invalid because Port has no outgoing connections</validationErrors>
</outputPorts>
<processors>
<id>9b696a42-d43a-304e-0000-000000000000</id>
<parentGroupId>82cb54fa-de00-38e0-0000-000000000000</parentGroupId>
<position>
<x>160.0</x>
<y>168.0</y>
</position>
<bundle>
<artifact>nifi-standard-nar</artifact>
<group>org.apache.nifi</group>
<version>1.13.2</version>
</bundle>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>Routing Strategy</key>
<value>
<name>Routing Strategy</name>
</value>
</entry>
<entry>
<key>conditionmatched</key>
<value>
<name>conditionmatched</name>
</value>
</entry>
</descriptors>
<executionNode>ALL</executionNode>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>Routing Strategy</key>
<value>Route to Property name</value>
</entry>
<entry>
<key>conditionmatched</key>
<value>${fileSize:gt( 1000 )}</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<executionNodeRestricted>false</executionNodeRestricted>
<name>RouteOnAttribute</name>
<relationships>
<autoTerminate>false</autoTerminate>
<name>conditionmatched</name>
</relationships>
<relationships>
<autoTerminate>true</autoTerminate>
<name>unmatched</name>
</relationships>
<state>STOPPED</state>
<style/>
<type>org.apache.nifi.processors.standard.RouteOnAttribute</type>
</processors>
</contents>
<flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
<flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<name>attrMatchCondition</name>
<variables/>
</processGroups>
</snippet>
<timestamp>06/25/2021 12:31:14 UTC</timestamp>
</template>
nodetypes/radon.nodes.datapipeline.process/attrMatchCondition/files/templateFile_localConn/attrmatchtemp_localConn.xml.mimetype 0 → 100644
+ 1
− 0
View file @ 7d88f18b
application/xml
\ No newline at end of file
nodetypes/radon.nodes.datapipeline.process/jsonValueMatchCondition/NodeType.tosca 0 → 100644
+ 75
− 0
View file @ 7d88f18b
tosca_definitions_version: tosca_simple_yaml_1_3
node_types:
radon.nodes.datapipeline.process.jsonValueMatchCondition:
derived_from: radon.nodes.datapipeline.process.LocalAction
metadata:
targetNamespace: "radon.nodes.datapipeline.process"
abstract: "false"
final: "false"
attributes:
template_name:
type: string
default: "jsonvaluematchtemp"
properties:
match_condition:
type: string
description: Comparison operator, supported, gt,equals,lt
default: "gt"
match_value:
type: string
default: 70
field_name:
type: string
default: "temperature"
interfaces:
Standard:
type: tosca.interfaces.node.lifecycle.Standard
operations:
create:
description: The standard create operation
inputs:
template_name:
type: string
required: true
default: { get_attribute: [ SELF, template_name ] }
template_file:
type: string
required: true
default: "jsonvaluematchtemp_localConn.xml"
implementation:
primary: create
dependencies: [ templateFile_localConn ]
timeout: 0
configure:
description: The standard configure operation
inputs:
pipeline_id:
type: string
required: true
default: { get_attribute: [ SELF, id ] }
match_condition:
type: string
required: true
default: { get_property: [ SELF, match_condition ] }
match_value:
type: string
required: true
default: { get_property: [ SELF, match_value ] }
field_name:
type: string
required: true
default: { get_property: [ SELF, field_name ] }
implementation:
primary: configure
timeout: 0
artifacts:
templateFile_localConn:
type: radon.artifacts.Ansible
file: jsonvaluematchtemp_localConn.xml
create:
type: radon.artifacts.Ansible
file: create.yml
configure:
type: radon.artifacts.Ansible
file: configure.yml
nodetypes/radon.nodes.datapipeline.process/jsonValueMatchCondition/appearance/bigIcon.png 0 → 100644
NaN GiB (NaN%)
View file @ 7d88f18b
File suppressed by a .gitattributes entry or the file's encoding is unsupported.
nodetypes/radon.nodes.datapipeline.process/jsonValueMatchCondition/appearance/bigIcon.png.mimetype 0 → 100644
+ 1
− 0
View file @ 7d88f18b
image/png
\ No newline at end of file
nodetypes/radon.nodes.datapipeline.process/jsonValueMatchCondition/files/configure/configure.yml 0 → 100644
+ 48
− 0
View file @ 7d88f18b
---
- hosts: all
vars:
get_id1: "processors[?component.name=='EvaluateJsonPath'].id"
get_id2: "processors[?component.name=='RouteOnAttribute'].id"
port: "{{ (lookup('file', '/tmp/nifi.json') | from_json) | default('8080', true) }}"
tasks:
- name: Get list of available NiFi processors within the process group
uri:
url: "http://localhost:{{ port }}/nifi-api/process-groups/{{ pipeline_id }}/processors"
method: Get
register: nifi_resources
- name: Configure the first processor within the process group
vars:
first_prcsr_id: "{{ nifi_resources.json | to_json | from_json | json_query(get_id1) | first}}"
schedulingPeriodValue: "{{ '0 sec' if schedulingStrategy == 'EVENT_DRIVEN' else schedulingPeriodCRON }}"
schedulingStrategyValue: "{{ 'TIMER_DRIVEN' if schedulingStrategy == 'EVENT_DRIVEN' else schedulingStrategy }}"
uri:
url: "http://localhost:{{ port }}/nifi-api/processors/{{ first_prcsr_id }}"
method: PUT
body:
"revision": {"version": 0}
"component":
"id": "{{ first_prcsr_id }}"
"config":
"properties": {
"matchValue":"$.{{field_name}}",
}
"schedulingStrategy": "{{ schedulingStrategyValue }}"
"schedulingPeriod": "{{ schedulingPeriodValue }}"
status_code: 200
body_format: json
- name: Configure the second processor within the process group
vars:
second_prcsr_id: "{{ nifi_resources.json | to_json | from_json | json_query(get_id2) | first}}"
uri:
url: "http://localhost:{{ port }}/nifi-api/processors/{{ second_prcsr_id }}"
method: PUT
body:
"revision": {"version": 0}
"component":
"id": "{{ second_prcsr_id }}"
"config":
"properties": {
"matchsuccess":"${matchValue:{{match_condition}}({{match_value}})}",
}
status_code: 200
body_format: json
nodetypes/radon.nodes.datapipeline.process/jsonValueMatchCondition/files/configure/configure.yml.mimetype 0 → 100644
+ 1
− 0
View file @ 7d88f18b
text/plain
\ No newline at end of file
nodetypes/radon.nodes.datapipeline.process/jsonValueMatchCondition/files/create/create.yml 0 → 100644
+ 44
− 0
View file @ 7d88f18b
---
- hosts: all
vars:
query: "resources[?name=='{{ template_name }}' && starts_with(identifier, '/templates/')].identifier"
port: "{{ (lookup('file', '/tmp/nifi.json') | from_json) | default('8080', true) }}"
tasks:
- name: Copy template file to remote system (shell command requires file to be in remote system)
copy:
src: "{{ template_file }}"
dest: template.xml
- name: Wait for port 8080 to become open on the host ( dont start checking for 4 seconds )
wait_for:
port: "{{ port }}"
timeout: 180
- name: upload template file to NiFi template repository
shell: 'curl -X "POST" "http://localhost:{{ port }}/nifi-api/process-groups/root/templates/upload" \
-H "Content-Type: multipart/form-data" \
-k \
--form template=@template.xml'
register: response
- name: Get list of available NiFi resources for fetching the id of the template by its name
uri:
url: http://localhost:{{ port }}/nifi-api/resources
method: Get
register: nifi_resources
- name: Initiate the template
vars:
template_id: "{{ nifi_resources.json | to_json | from_json | json_query(query) | first }}"
uri:
url: http://localhost:{{ port }}/nifi-api/process-groups/root/template-instance
method: POST
body:
"originX": 2.0
"originY": 3.0
"templateId": "{{ template_id.split('/')[2] }}"
status_code: 201
body_format: json
register: pipeline_info
- name: Set pipeline id property value (process-groups)
set_stats:
data:
id: "{{ pipeline_info.json.flow.processGroups[0].id }}"
# pipeline_type: "processGroups"
when: pipeline_info.json.flow.processGroups|length > 0
nodetypes/radon.nodes.datapipeline.process/jsonValueMatchCondition/files/create/create.yml.mimetype 0 → 100644
+ 1
− 0
View file @ 7d88f18b
text/plain
\ No newline at end of file
nodetypes/radon.nodes.datapipeline.process/jsonValueMatchCondition/files/templateFile_localConn/jsonvaluematchtemp_localConn.xml 0 → 100644
+ 284
− 0
View file @ 7d88f18b
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<template encoding-version="1.3">
<description></description>
<groupId>eb3aee6c-0179-1000-acd4-0e00ef9b3344</groupId>
<name>jsonvaluematchtemp</name>
<snippet>
<processGroups>
<id>1519d1f2-d296-3fe3-0000-000000000000</id>
<parentGroupId>b7905d91-01ef-3484-0000-000000000000</parentGroupId>
<position>
<x>0.0</x>
<y>0.0</y>
</position>
<comments></comments>
<contents>
<connections>
<id>1e16f742-7c17-3b4d-0000-000000000000</id>
<parentGroupId>1519d1f2-d296-3fe3-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>1519d1f2-d296-3fe3-0000-000000000000</groupId>
<id>ed9164cd-7685-3573-0000-000000000000</id>
<type>PROCESSOR</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
<loadBalancePartitionAttribute></loadBalancePartitionAttribute>
<loadBalanceStatus>LOAD_BALANCE_NOT_CONFIGURED</loadBalanceStatus>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<name></name>
<selectedRelationships>matched</selectedRelationships>
<source>
<groupId>1519d1f2-d296-3fe3-0000-000000000000</groupId>
<id>a9377ddc-0c41-3125-0000-000000000000</id>
<type>PROCESSOR</type>
</source>
<zIndex>0</zIndex>
</connections>
<connections>
<id>349c352a-0100-311e-0000-000000000000</id>
<parentGroupId>1519d1f2-d296-3fe3-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>1519d1f2-d296-3fe3-0000-000000000000</groupId>
<id>591ec843-81f5-3cc3-0000-000000000000</id>
<type>OUTPUT_PORT</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
<loadBalancePartitionAttribute></loadBalancePartitionAttribute>
<loadBalanceStatus>LOAD_BALANCE_NOT_CONFIGURED</loadBalanceStatus>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<name></name>
<selectedRelationships>matchsuccess</selectedRelationships>
<source>
<groupId>1519d1f2-d296-3fe3-0000-000000000000</groupId>
<id>ed9164cd-7685-3573-0000-000000000000</id>
<type>PROCESSOR</type>
</source>
<zIndex>0</zIndex>
</connections>
<connections>
<id>ba53cf33-48dc-3f3a-0000-000000000000</id>
<parentGroupId>1519d1f2-d296-3fe3-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>1519d1f2-d296-3fe3-0000-000000000000</groupId>
<id>a9377ddc-0c41-3125-0000-000000000000</id>
<type>PROCESSOR</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
<loadBalancePartitionAttribute></loadBalancePartitionAttribute>
<loadBalanceStatus>LOAD_BALANCE_NOT_CONFIGURED</loadBalanceStatus>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<name></name>
<source>
<groupId>1519d1f2-d296-3fe3-0000-000000000000</groupId>
<id>4e64e7ac-888d-3677-0000-000000000000</id>
<type>INPUT_PORT</type>
</source>
<zIndex>0</zIndex>
</connections>
<inputPorts>
<id>4e64e7ac-888d-3677-0000-000000000000</id>
<parentGroupId>1519d1f2-d296-3fe3-0000-000000000000</parentGroupId>
<position>
<x>216.0</x>
<y>-8.0</y>
</position>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<name>jsonval_IP_local</name>
<state>STOPPED</state>
<type>INPUT_PORT</type>
<validationErrors>'Port 'jsonval_IP_local'' is invalid because Port has no incoming connections</validationErrors>
</inputPorts>
<outputPorts>
<id>591ec843-81f5-3cc3-0000-000000000000</id>
<parentGroupId>1519d1f2-d296-3fe3-0000-000000000000</parentGroupId>
<position>
<x>1048.0</x>
<y>616.0</y>
</position>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<name>jsonval_OP_local</name>
<state>STOPPED</state>
<type>OUTPUT_PORT</type>
<validationErrors>'Port 'jsonval_OP_local'' is invalid because Port has no outgoing connections</validationErrors>
</outputPorts>
<processors>
<id>a9377ddc-0c41-3125-0000-000000000000</id>
<parentGroupId>1519d1f2-d296-3fe3-0000-000000000000</parentGroupId>
<position>
<x>360.0</x>
<y>128.0</y>
</position>
<bundle>
<artifact>nifi-standard-nar</artifact>
<group>org.apache.nifi</group>
<version>1.13.2</version>
</bundle>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>Destination</key>
<value>
<name>Destination</name>
</value>
</entry>
<entry>
<key>Return Type</key>
<value>
<name>Return Type</name>
</value>
</entry>
<entry>
<key>Path Not Found Behavior</key>
<value>
<name>Path Not Found Behavior</name>
</value>
</entry>
<entry>
<key>Null Value Representation</key>
<value>
<name>Null Value Representation</name>
</value>
</entry>
<entry>
<key>matchValue</key>
<value>
<name>matchValue</name>
</value>
</entry>
</descriptors>
<executionNode>ALL</executionNode>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>Destination</key>
<value>flowfile-attribute</value>
</entry>
<entry>
<key>Return Type</key>
<value>auto-detect</value>
</entry>
<entry>
<key>Path Not Found Behavior</key>
<value>ignore</value>
</entry>
<entry>
<key>Null Value Representation</key>
<value>empty string</value>
</entry>
<entry>
<key>matchValue</key>
<value>$.temp</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<executionNodeRestricted>false</executionNodeRestricted>
<name>EvaluateJsonPath</name>
<relationships>
<autoTerminate>true</autoTerminate>
<name>failure</name>
</relationships>
<relationships>
<autoTerminate>false</autoTerminate>
<name>matched</name>
</relationships>
<relationships>
<autoTerminate>true</autoTerminate>
<name>unmatched</name>
</relationships>
<state>STOPPED</state>
<style/>
<type>org.apache.nifi.processors.standard.EvaluateJsonPath</type>
</processors>
<processors>
<id>ed9164cd-7685-3573-0000-000000000000</id>
<parentGroupId>1519d1f2-d296-3fe3-0000-000000000000</parentGroupId>
<position>
<x>704.0</x>
<y>360.0</y>
</position>
<bundle>
<artifact>nifi-standard-nar</artifact>
<group>org.apache.nifi</group>
<version>1.13.2</version>
</bundle>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>Routing Strategy</key>
<value>
<name>Routing Strategy</name>
</value>
</entry>
<entry>
<key>matchsuccess</key>
<value>
<name>matchsuccess</name>
</value>
</entry>
</descriptors>
<executionNode>ALL</executionNode>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>Routing Strategy</key>
<value>Route to Property name</value>
</entry>
<entry>
<key>matchsuccess</key>
<value>${matchValue:gt(50)}</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<executionNodeRestricted>false</executionNodeRestricted>
<name>RouteOnAttribute</name>
<relationships>
<autoTerminate>false</autoTerminate>
<name>matchsuccess</name>
</relationships>
<relationships>
<autoTerminate>true</autoTerminate>
<name>unmatched</name>
</relationships>
<state>STOPPED</state>
<style/>
<type>org.apache.nifi.processors.standard.RouteOnAttribute</type>
</processors>
</contents>
<flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
<flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<name>jsonValueMatchCondition</name>
<variables/>
</processGroups>
</snippet>
<timestamp>06/25/2021 12:31:50 UTC</timestamp>
</template>
nodetypes/radon.nodes.datapipeline.process/jsonValueMatchCondition/files/templateFile_localConn/jsonvaluematchtemp_localConn.xml.mimetype 0 → 100644
+ 1
− 0
View file @ 7d88f18b
application/xml
\ No newline at end of file
servicetemplates/radon.blueprints.examples/ConditionalDataFiltering/ServiceTemplate.tosca 0 → 100644
+ 154
− 0
View file @ 7d88f18b
tosca_definitions_version: tosca_simple_yaml_1_3
metadata:
targetNamespace: "radon.blueprints.examples"
topology_template:
node_templates:
attrMatchCondition_0:
type: radon.nodes.datapipeline.process.attrMatchCondition
metadata:
x: "665"
y: "107"
displayName: "attrMatchCondition"
properties:
matchvalue: 1000
matchcondition: "gt"
schedulingStrategy: "EVENT_DRIVEN"
matchkey: "filesize"
schedulingPeriodCRON: "* * * * * ?"
requirements:
- ConnectToPipeline:
node: PublishLocal_0
relationship: con_ConnectNifiLocal_1
capability: ConnectToPipeline
- host:
node: Nifi_0
relationship: con_HostedOn_5
capability: host
PublishLocal_0:
type: radon.nodes.datapipeline.destination.PublishLocal
metadata:
x: "1294"
y: "107"
displayName: "PublishLocal"
properties:
create_if_missing: true
schedulingStrategy: "EVENT_DRIVEN"
schedulingPeriodCRON: "* * * * * ?"
name: "adsad"
directory: "/tmp/out1"
requirements:
- host:
node: Nifi_0
relationship: con_HostedOn_4
capability: host
jsonValueMatchCondition_0:
type: radon.nodes.datapipeline.process.jsonValueMatchCondition
metadata:
x: "667"
y: "268"
displayName: "jsonValueMatchCondition"
properties:
schedulingStrategy: "EVENT_DRIVEN"
schedulingPeriodCRON: "* * * * * ?"
match_condition: "gt"
match_value: 70
field_name: "temperature"
requirements:
- ConnectToPipeline:
node: PublishLocal_1
relationship: con_ConnectNifiLocal_0
capability: ConnectToPipeline
- host:
node: Nifi_0
relationship: con_HostedOn_2
capability: host
PublishLocal_1:
type: radon.nodes.datapipeline.destination.PublishLocal
metadata:
x: "1361"
y: "267"
displayName: "PublishLocal"
properties:
create_if_missing: true
schedulingStrategy: "EVENT_DRIVEN"
schedulingPeriodCRON: "* * * * * ?"
name: "asdasd"
directory: "/tmp/iout2"
requirements:
- host:
node: Nifi_0
relationship: con_HostedOn_3
capability: host
ConsumeLocal_0:
type: radon.nodes.datapipeline.source.ConsumeLocal
metadata:
x: "105"
y: "161"
displayName: "ConsumeLocal"
properties:
schedulingStrategy: "EVENT_DRIVEN"
schedulingPeriodCRON: "* * * * * ?"
directory: "/tmp/input"
requirements:
- connectToPipeline:
node: attrMatchCondition_0
relationship: con_ConnectNifiLocal_2
capability: ConnectToPipeline
- connectToPipeline:
node: jsonValueMatchCondition_0
relationship: con_ConnectNifiLocal_3
capability: ConnectToPipeline
- host:
node: Nifi_0
relationship: con_HostedOn_1
capability: host
OpenStack_0:
type: radon.nodes.VM.OpenStack
metadata:
x: "1275"
y: "570"
displayName: "OpenStack"
properties:
flavor: "c1.medium"
key_name: "jakovits_ldpc"
image: "21cd7486-a2a6-458c-9b05-9207c0b63819"
ssh_username: "ubuntu"
name: "sdfsdf"
network: "provider_64_net"
Nifi_0:
type: radon.nodes.nifi.Nifi
metadata:
x: "854"
y: "556"
displayName: "Nifi"
properties:
port: 8080
webinterface_public: true
component_version: "1.13.2"
requirements:
- host:
node: OpenStack_0
relationship: con_HostedOn_0
capability: host
relationship_templates:
con_HostedOn_2:
type: tosca.relationships.HostedOn
con_ConnectNifiLocal_3:
type: radon.relationships.datapipeline.ConnectNifiLocal
con_HostedOn_3:
type: tosca.relationships.HostedOn
con_HostedOn_0:
type: tosca.relationships.HostedOn
con_HostedOn_1:
type: tosca.relationships.HostedOn
con_ConnectNifiLocal_0:
type: radon.relationships.datapipeline.ConnectNifiLocal
con_ConnectNifiLocal_2:
type: radon.relationships.datapipeline.ConnectNifiLocal
con_ConnectNifiLocal_1:
type: radon.relationships.datapipeline.ConnectNifiLocal
con_HostedOn_4:
type: tosca.relationships.HostedOn
con_HostedOn_5:
type: tosca.relationships.HostedOn
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment