Skip to content
Snippets Groups Projects
Commit c58ed8ba authored by pjakovits's avatar pjakovits
Browse files

Add data pipeline node types for compression

parent 938ca68a
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 815 additions and 0 deletions
nodetypes/radon.nodes.datapipeline.process/compressData/NodeType.tosca 0 → 100644
+ 61
− 0
View file @ c58ed8ba
tosca_definitions_version: tosca_simple_yaml_1_3
node_types:
radon.nodes.datapipeline.process.compressData:
derived_from: radon.nodes.datapipeline.process.LocalAction
metadata:
targetNamespace: "radon.nodes.datapipeline.process"
abstract: "false"
final: "false"
attributes:
template_name:
type: string
default: "CompressDataTemplate"
properties:
compression_format:
type: string
description: Compression format (gzip, snappy,lzma, bzip2)
default: "gzip"
interfaces:
Standard:
type: tosca.interfaces.node.lifecycle.Standard
operations:
create:
description: The standard create operation
inputs:
template_name:
type: string
required: true
default: { get_attribute: [ SELF, template_name ] }
template_file:
type: string
required: true
default: "compresstemplate_localConn.xml"
implementation:
primary: create
dependencies: [ templateFile_localConn ]
timeout: 0
configure:
description: The standard configure operation
inputs:
compression_format:
type: string
required: true
default: { get_property: [ SELF, compression_format ] }
pipeline_id:
type: string
required: true
default: { get_attribute: [ SELF, id ] }
implementation:
primary: configure
timeout: 0
artifacts:
templateFile_localConn:
type: radon.artifacts.Ansible
file: compresstemplate_localConn.xml
create:
type: radon.artifacts.Ansible
file: create.yml
configure:
type: radon.artifacts.Ansible
file: configure.yml
nodetypes/radon.nodes.datapipeline.process/compressData/appearance/bigIcon.png 0 → 100644
NaN GiB (NaN%)
View file @ c58ed8ba
File suppressed by a .gitattributes entry or the file's encoding is unsupported.
nodetypes/radon.nodes.datapipeline.process/compressData/appearance/bigIcon.png.mimetype 0 → 100644
+ 1
− 0
View file @ c58ed8ba
image/png
\ No newline at end of file
nodetypes/radon.nodes.datapipeline.process/compressData/files/configure/configure.yml 0 → 100644
+ 31
− 0
View file @ c58ed8ba
---
- hosts: all
vars:
get_id1: "processors[0].id"
port: "{{ (lookup('file', '/tmp/nifi.json') | from_json) | default('8080', true) }}"
tasks:
- name: Get list of available NiFi processors within the process group
uri:
url: "http://localhost:{{ port }}/nifi-api/process-groups/{{ pipeline_id }}/processors"
method: Get
register: nifi_resources
- name: Configure the first processor within the process group
vars:
first_prcsr_id: "{{ nifi_resources.json | to_json | from_json | json_query(get_id1) }}"
schedulingPeriodValue: "{{ '0 sec' if schedulingStrategy == 'EVENT_DRIVEN' else schedulingPeriodCRON }}"
schedulingStrategyValue: "{{ 'TIMER_DRIVEN' if schedulingStrategy == 'EVENT_DRIVEN' else schedulingStrategy }}"
uri:
url: "http://localhost:{{ port }}/nifi-api/processors/{{ first_prcsr_id }}"
method: PUT
body:
"revision": {"version": 0}
"component":
"id": "{{ first_prcsr_id }}"
"config":
"properties": {
"Compression Format":"{{compression_format}}"
}
"schedulingStrategy": "{{ schedulingStrategyValue }}"
"schedulingPeriod": "{{ schedulingPeriodValue }}"
status_code: 200
body_format: json
nodetypes/radon.nodes.datapipeline.process/compressData/files/configure/configure.yml.mimetype 0 → 100644
+ 1
− 0
View file @ c58ed8ba
text/plain
\ No newline at end of file
nodetypes/radon.nodes.datapipeline.process/compressData/files/create/create.yml 0 → 100644
+ 44
− 0
View file @ c58ed8ba
---
- hosts: all
vars:
query: "resources[?name=='{{ template_name }}' && starts_with(identifier, '/templates/')].identifier"
port: "{{ (lookup('file', '/tmp/nifi.json') | from_json) | default('8080', true) }}"
tasks:
- name: Copy template file to remote system (shell command requires file to be in remote system)
copy:
src: "{{ template_file }}"
dest: template.xml
- name: Wait for port 8080 to become open on the host ( dont start checking for 4 seconds )
wait_for:
port: "{{ port }}"
timeout: 180
- name: upload template file to NiFi template repository
shell: 'curl -X "POST" "http://localhost:{{ port }}/nifi-api/process-groups/root/templates/upload" \
-H "Content-Type: multipart/form-data" \
-k \
--form template=@template.xml'
register: response
- name: Get list of available NiFi resources for fetching the id of the template by its name
uri:
url: http://localhost:{{ port }}/nifi-api/resources
method: Get
register: nifi_resources
- name: Initiate the template
vars:
template_id: "{{ nifi_resources.json | to_json | from_json | json_query(query) | first }}"
uri:
url: http://localhost:{{ port }}/nifi-api/process-groups/root/template-instance
method: POST
body:
"originX": 2.0
"originY": 3.0
"templateId": "{{ template_id.split('/')[2] }}"
status_code: 201
body_format: json
register: pipeline_info
- name: Set pipeline id property value (process-groups)
set_stats:
data:
id: "{{ pipeline_info.json.flow.processGroups[0].id }}"
# pipeline_type: "processGroups"
when: pipeline_info.json.flow.processGroups|length > 0
nodetypes/radon.nodes.datapipeline.process/compressData/files/create/create.yml.mimetype 0 → 100644
+ 1
− 0
View file @ c58ed8ba
text/plain
\ No newline at end of file
nodetypes/radon.nodes.datapipeline.process/compressData/files/templateFile_localConn/compresstemplate_localConn.xml 0 → 100644
+ 193
− 0
View file @ c58ed8ba
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<template encoding-version="1.3">
<description></description>
<groupId>eb3aee6c-0179-1000-acd4-0e00ef9b3344</groupId>
<name>CompressDataTemplate</name>
<snippet>
<processGroups>
<id>7abdde99-4505-350e-0000-000000000000</id>
<parentGroupId>b7905d91-01ef-3484-0000-000000000000</parentGroupId>
<position>
<x>0.0</x>
<y>0.0</y>
</position>
<comments></comments>
<contents>
<connections>
<id>5d7b614a-30e7-3986-0000-000000000000</id>
<parentGroupId>7abdde99-4505-350e-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>7abdde99-4505-350e-0000-000000000000</groupId>
<id>56524ffb-59dd-3326-0000-000000000000</id>
<type>OUTPUT_PORT</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
<loadBalancePartitionAttribute></loadBalancePartitionAttribute>
<loadBalanceStatus>LOAD_BALANCE_NOT_CONFIGURED</loadBalanceStatus>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<name></name>
<selectedRelationships>success</selectedRelationships>
<source>
<groupId>7abdde99-4505-350e-0000-000000000000</groupId>
<id>22858236-bb57-3f7a-0000-000000000000</id>
<type>PROCESSOR</type>
</source>
<zIndex>0</zIndex>
</connections>
<connections>
<id>7553f973-9d8d-387f-0000-000000000000</id>
<parentGroupId>7abdde99-4505-350e-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>7abdde99-4505-350e-0000-000000000000</groupId>
<id>22858236-bb57-3f7a-0000-000000000000</id>
<type>PROCESSOR</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
<loadBalancePartitionAttribute></loadBalancePartitionAttribute>
<loadBalanceStatus>LOAD_BALANCE_NOT_CONFIGURED</loadBalanceStatus>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<name></name>
<source>
<groupId>7abdde99-4505-350e-0000-000000000000</groupId>
<id>ebee89b3-d0c7-319c-0000-000000000000</id>
<type>INPUT_PORT</type>
</source>
<zIndex>0</zIndex>
</connections>
<inputPorts>
<id>ebee89b3-d0c7-319c-0000-000000000000</id>
<parentGroupId>7abdde99-4505-350e-0000-000000000000</parentGroupId>
<position>
<x>608.0</x>
<y>88.0</y>
</position>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<name>compress_IP_local</name>
<state>STOPPED</state>
<type>INPUT_PORT</type>
<validationErrors>'Port 'compress_IP_local'' is invalid because Port has no incoming connections</validationErrors>
</inputPorts>
<outputPorts>
<id>56524ffb-59dd-3326-0000-000000000000</id>
<parentGroupId>7abdde99-4505-350e-0000-000000000000</parentGroupId>
<position>
<x>600.0</x>
<y>624.0</y>
</position>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<name>compress_OP_local</name>
<state>STOPPED</state>
<type>OUTPUT_PORT</type>
<validationErrors>'Port 'compress_OP_local'' is invalid because Port has no outgoing connections</validationErrors>
</outputPorts>
<processors>
<id>22858236-bb57-3f7a-0000-000000000000</id>
<parentGroupId>7abdde99-4505-350e-0000-000000000000</parentGroupId>
<position>
<x>544.0</x>
<y>328.0</y>
</position>
<bundle>
<artifact>nifi-standard-nar</artifact>
<group>org.apache.nifi</group>
<version>1.13.2</version>
</bundle>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>Mode</key>
<value>
<name>Mode</name>
</value>
</entry>
<entry>
<key>Compression Format</key>
<value>
<name>Compression Format</name>
</value>
</entry>
<entry>
<key>Compression Level</key>
<value>
<dependencies>
<dependentValues>use mime.type attribute</dependentValues>
<dependentValues>deflate</dependentValues>
<dependentValues>gzip</dependentValues>
<dependentValues>xz-lzma2</dependentValues>
<propertyName>Compression Format</propertyName>
</dependencies>
<dependencies>
<dependentValues>compress</dependentValues>
<propertyName>Mode</propertyName>
</dependencies>
<name>Compression Level</name>
</value>
</entry>
<entry>
<key>Update Filename</key>
<value>
<name>Update Filename</name>
</value>
</entry>
</descriptors>
<executionNode>ALL</executionNode>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>Mode</key>
<value>compress</value>
</entry>
<entry>
<key>Compression Format</key>
<value>gzip</value>
</entry>
<entry>
<key>Compression Level</key>
<value>1</value>
</entry>
<entry>
<key>Update Filename</key>
<value>true</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<executionNodeRestricted>false</executionNodeRestricted>
<name>CompressContent</name>
<relationships>
<autoTerminate>true</autoTerminate>
<name>failure</name>
</relationships>
<relationships>
<autoTerminate>false</autoTerminate>
<name>success</name>
</relationships>
<state>STOPPED</state>
<style/>
<type>org.apache.nifi.processors.standard.CompressContent</type>
</processors>
</contents>
<flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
<flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<name>CompressData</name>
<variables/>
</processGroups>
</snippet>
<timestamp>06/30/2021 07:49:57 UTC</timestamp>
</template>
nodetypes/radon.nodes.datapipeline.process/compressData/files/templateFile_localConn/compresstemplate_localConn.xml.mimetype 0 → 100644
+ 1
− 0
View file @ c58ed8ba
application/xml
\ No newline at end of file
nodetypes/radon.nodes.datapipeline.process/unCompressData/NodeType.tosca 0 → 100644
+ 61
− 0
View file @ c58ed8ba
tosca_definitions_version: tosca_simple_yaml_1_3
node_types:
radon.nodes.datapipeline.process.unCompressData:
derived_from: radon.nodes.datapipeline.process.LocalAction
metadata:
targetNamespace: "radon.nodes.datapipeline.process"
abstract: "false"
final: "false"
attributes:
template_name:
type: string
default: "unCompressDataTemplate"
properties:
compression_format:
type: string
description: Compression format (gzip, snappy,lzma, bzip2)
default: "gzip"
interfaces:
Standard:
type: tosca.interfaces.node.lifecycle.Standard
operations:
create:
description: The standard create operation
inputs:
template_name:
type: string
required: true
default: { get_attribute: [ SELF, template_name ] }
template_file:
type: string
required: true
default: "uncompresstemplate_localConn.xml"
implementation:
primary: create
dependencies: [ templateFile_localConn ]
timeout: 0
configure:
description: The standard configure operation
inputs:
compression_format:
type: string
required: true
default: { get_property: [ SELF, compression_format ] }
pipeline_id:
type: string
required: true
default: { get_attribute: [ SELF, id ] }
implementation:
primary: configure
timeout: 0
artifacts:
templateFile_localConn:
type: radon.artifacts.Ansible
file: uncompresstemplate_localConn.xml
create:
type: radon.artifacts.Ansible
file: create.yml
configure:
type: radon.artifacts.Ansible
file: configure.yml
nodetypes/radon.nodes.datapipeline.process/unCompressData/appearance/bigIcon.png 0 → 100644
NaN GiB (NaN%)
View file @ c58ed8ba
File suppressed by a .gitattributes entry or the file's encoding is unsupported.
nodetypes/radon.nodes.datapipeline.process/unCompressData/appearance/bigIcon.png.mimetype 0 → 100644
+ 1
− 0
View file @ c58ed8ba
image/png
\ No newline at end of file
nodetypes/radon.nodes.datapipeline.process/unCompressData/files/configure/configure.yml 0 → 100644
+ 31
− 0
View file @ c58ed8ba
---
- hosts: all
vars:
get_id1: "processors[0].id"
port: "{{ (lookup('file', '/tmp/nifi.json') | from_json) | default('8080', true) }}"
tasks:
- name: Get list of available NiFi processors within the process group
uri:
url: "http://localhost:{{ port }}/nifi-api/process-groups/{{ pipeline_id }}/processors"
method: Get
register: nifi_resources
- name: Configure the first processor within the process group
vars:
first_prcsr_id: "{{ nifi_resources.json | to_json | from_json | json_query(get_id1) }}"
schedulingPeriodValue: "{{ '0 sec' if schedulingStrategy == 'EVENT_DRIVEN' else schedulingPeriodCRON }}"
schedulingStrategyValue: "{{ 'TIMER_DRIVEN' if schedulingStrategy == 'EVENT_DRIVEN' else schedulingStrategy }}"
uri:
url: "http://localhost:{{ port }}/nifi-api/processors/{{ first_prcsr_id }}"
method: PUT
body:
"revision": {"version": 0}
"component":
"id": "{{ first_prcsr_id }}"
"config":
"properties": {
"Compression Format":"{{compression_format}}"
}
"schedulingStrategy": "{{ schedulingStrategyValue }}"
"schedulingPeriod": "{{ schedulingPeriodValue }}"
status_code: 200
body_format: json
nodetypes/radon.nodes.datapipeline.process/unCompressData/files/configure/configure.yml.mimetype 0 → 100644
+ 1
− 0
View file @ c58ed8ba
text/plain
\ No newline at end of file
nodetypes/radon.nodes.datapipeline.process/unCompressData/files/create/create.yml 0 → 100644
+ 44
− 0
View file @ c58ed8ba
---
- hosts: all
vars:
query: "resources[?name=='{{ template_name }}' && starts_with(identifier, '/templates/')].identifier"
port: "{{ (lookup('file', '/tmp/nifi.json') | from_json) | default('8080', true) }}"
tasks:
- name: Copy template file to remote system (shell command requires file to be in remote system)
copy:
src: "{{ template_file }}"
dest: template.xml
- name: Wait for port 8080 to become open on the host ( dont start checking for 4 seconds )
wait_for:
port: "{{ port }}"
timeout: 180
- name: upload template file to NiFi template repository
shell: 'curl -X "POST" "http://localhost:{{ port }}/nifi-api/process-groups/root/templates/upload" \
-H "Content-Type: multipart/form-data" \
-k \
--form template=@template.xml'
register: response
- name: Get list of available NiFi resources for fetching the id of the template by its name
uri:
url: http://localhost:{{ port }}/nifi-api/resources
method: Get
register: nifi_resources
- name: Initiate the template
vars:
template_id: "{{ nifi_resources.json | to_json | from_json | json_query(query) | first }}"
uri:
url: http://localhost:{{ port }}/nifi-api/process-groups/root/template-instance
method: POST
body:
"originX": 2.0
"originY": 3.0
"templateId": "{{ template_id.split('/')[2] }}"
status_code: 201
body_format: json
register: pipeline_info
- name: Set pipeline id property value (process-groups)
set_stats:
data:
id: "{{ pipeline_info.json.flow.processGroups[0].id }}"
# pipeline_type: "processGroups"
when: pipeline_info.json.flow.processGroups|length > 0
nodetypes/radon.nodes.datapipeline.process/unCompressData/files/create/create.yml.mimetype 0 → 100644
+ 1
− 0
View file @ c58ed8ba
text/plain
\ No newline at end of file
nodetypes/radon.nodes.datapipeline.process/unCompressData/files/templateFile_localConn/uncompresstemplate_localConn.xml 0 → 100644
+ 192
− 0
View file @ c58ed8ba
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<template encoding-version="1.3">
<description></description>
<groupId>eb3aee6c-0179-1000-acd4-0e00ef9b3344</groupId>
<name>unCompressDataTemplate</name>
<snippet>
<processGroups>
<id>a0411824-77fc-3a78-0000-000000000000</id>
<parentGroupId>b7905d91-01ef-3484-0000-000000000000</parentGroupId>
<position>
<x>0.0</x>
<y>0.0</y>
</position>
<comments></comments>
<contents>
<connections>
<id>1912bd29-9609-3a8e-0000-000000000000</id>
<parentGroupId>a0411824-77fc-3a78-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>a0411824-77fc-3a78-0000-000000000000</groupId>
<id>a7c908c1-66bb-3eb1-0000-000000000000</id>
<type>PROCESSOR</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
<loadBalancePartitionAttribute></loadBalancePartitionAttribute>
<loadBalanceStatus>LOAD_BALANCE_NOT_CONFIGURED</loadBalanceStatus>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<name></name>
<source>
<groupId>a0411824-77fc-3a78-0000-000000000000</groupId>
<id>3cb6cc7e-c9cb-39a0-0000-000000000000</id>
<type>INPUT_PORT</type>
</source>
<zIndex>0</zIndex>
</connections>
<connections>
<id>3c039fab-96c2-3f47-0000-000000000000</id>
<parentGroupId>a0411824-77fc-3a78-0000-000000000000</parentGroupId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>a0411824-77fc-3a78-0000-000000000000</groupId>
<id>2cf582ac-3f5d-3884-0000-000000000000</id>
<type>OUTPUT_PORT</type>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
<loadBalancePartitionAttribute></loadBalancePartitionAttribute>
<loadBalanceStatus>LOAD_BALANCE_NOT_CONFIGURED</loadBalanceStatus>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<name></name>
<selectedRelationships>success</selectedRelationships>
<source>
<groupId>a0411824-77fc-3a78-0000-000000000000</groupId>
<id>a7c908c1-66bb-3eb1-0000-000000000000</id>
<type>PROCESSOR</type>
</source>
<zIndex>0</zIndex>
</connections>
<inputPorts>
<id>3cb6cc7e-c9cb-39a0-0000-000000000000</id>
<parentGroupId>a0411824-77fc-3a78-0000-000000000000</parentGroupId>
<position>
<x>625.9999544358313</x>
<y>172.9999933312421</y>
</position>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<name>uncompress_IP_local</name>
<state>STOPPED</state>
<type>INPUT_PORT</type>
<validationErrors>'Port 'uncompress_IP_local'' is invalid because Port has no incoming connections</validationErrors>
</inputPorts>
<outputPorts>
<id>2cf582ac-3f5d-3884-0000-000000000000</id>
<parentGroupId>a0411824-77fc-3a78-0000-000000000000</parentGroupId>
<position>
<x>660.9999544358313</x>
<y>524.9999933312421</y>
</position>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<name>uncompress_OP_local</name>
<state>STOPPED</state>
<type>OUTPUT_PORT</type>
<validationErrors>'Port 'uncompress_OP_local'' is invalid because Port has no outgoing connections</validationErrors>
</outputPorts>
<processors>
<id>a7c908c1-66bb-3eb1-0000-000000000000</id>
<parentGroupId>a0411824-77fc-3a78-0000-000000000000</parentGroupId>
<position>
<x>600.0</x>
<y>312.0</y>
</position>
<bundle>
<artifact>nifi-standard-nar</artifact>
<group>org.apache.nifi</group>
<version>1.13.2</version>
</bundle>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>Mode</key>
<value>
<name>Mode</name>
</value>
</entry>
<entry>
<key>Compression Format</key>
<value>
<name>Compression Format</name>
</value>
</entry>
<entry>
<key>Compression Level</key>
<value>
<dependencies>
<dependentValues>use mime.type attribute</dependentValues>
<dependentValues>deflate</dependentValues>
<dependentValues>gzip</dependentValues>
<dependentValues>xz-lzma2</dependentValues>
<propertyName>Compression Format</propertyName>
</dependencies>
<dependencies>
<dependentValues>compress</dependentValues>
<propertyName>Mode</propertyName>
</dependencies>
<name>Compression Level</name>
</value>
</entry>
<entry>
<key>Update Filename</key>
<value>
<name>Update Filename</name>
</value>
</entry>
</descriptors>
<executionNode>ALL</executionNode>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>Mode</key>
<value>decompress</value>
</entry>
<entry>
<key>Compression Format</key>
<value>gzip</value>
</entry>
<entry>
<key>Compression Level</key>
<value>1</value>
</entry>
<entry>
<key>Update Filename</key>
<value>true</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<executionNodeRestricted>false</executionNodeRestricted>
<name>CompressContent</name>
<relationships>
<autoTerminate>true</autoTerminate>
<name>failure</name>
</relationships>
<relationships>
<autoTerminate>false</autoTerminate>
<name>success</name>
</relationships>
<state>STOPPED</state>
<style/>
<type>org.apache.nifi.processors.standard.CompressContent</type>
</processors>
</contents>
<flowfileConcurrency>UNBOUNDED</flowfileConcurrency>
<flowfileOutboundPolicy>STREAM_WHEN_AVAILABLE</flowfileOutboundPolicy>
<name>unCompressData</name>
<variables/>
</processGroups>
</snippet>
<timestamp>06/30/2021 07:50:12 UTC</timestamp>
</template>
nodetypes/radon.nodes.datapipeline.process/unCompressData/files/templateFile_localConn/uncompresstemplate_localConn.xml.mimetype 0 → 100644
+ 1
− 0
View file @ c58ed8ba
application/xml
\ No newline at end of file
servicetemplates/radon.blueprints.examples/DataPipelineCompressionExample/ServiceTemplate.tosca 0 → 100644
+ 150
− 0
View file @ c58ed8ba
tosca_definitions_version: tosca_simple_yaml_1_3
metadata:
targetNamespace: "radon.blueprints.examples"
topology_template:
node_templates:
PublishLocal_0:
type: radon.nodes.datapipeline.destination.PublishLocal
metadata:
x: "1562"
y: "96"
displayName: "PublishLocal"
properties:
create_if_missing: true
schedulingStrategy: "EVENT_DRIVEN"
schedulingPeriodCRON: "* * * * * ?"
name: "adsad"
directory: "/tmp/out1"
requirements:
- host:
node: Nifi_0
relationship: con_HostedOn_4
capability: host
PublishLocal_1:
type: radon.nodes.datapipeline.destination.PublishLocal
metadata:
x: "1171"
y: "388"
displayName: "PublishLocal"
properties:
create_if_missing: true
schedulingStrategy: "EVENT_DRIVEN"
schedulingPeriodCRON: "* * * * * ?"
name: "asdasd"
directory: "/tmp/iout2"
requirements:
- host:
node: Nifi_0
relationship: con_HostedOn_3
capability: host
compressData_0:
type: radon.nodes.datapipeline.process.compressData
metadata:
x: "483"
y: "100"
displayName: "compressData"
properties:
schedulingStrategy: "EVENT_DRIVEN"
compression_format: "gzip"
schedulingPeriodCRON: "* * * * * ?"
requirements:
- ConnectToPipeline:
node: unCompressData_0
relationship: con_ConnectNifiLocal_1
capability: ConnectToPipeline
- ConnectToPipeline:
node: PublishLocal_1
relationship: con_ConnectNifiLocal_3
capability: ConnectToPipeline
- host:
node: Nifi_0
relationship: con_HostedOn_5
capability: host
unCompressData_0:
type: radon.nodes.datapipeline.process.unCompressData
metadata:
x: "995"
y: "79"
displayName: "unCompressData"
properties:
schedulingStrategy: "EVENT_DRIVEN"
compression_format: "gzip"
schedulingPeriodCRON: "* * * * * ?"
requirements:
- ConnectToPipeline:
node: PublishLocal_0
relationship: con_ConnectNifiLocal_2
capability: ConnectToPipeline
- host:
node: Nifi_0
relationship: con_HostedOn_2
capability: host
ConsumeLocal_0:
type: radon.nodes.datapipeline.source.ConsumeLocal
metadata:
x: "78"
y: "168"
displayName: "ConsumeLocal"
properties:
schedulingStrategy: "EVENT_DRIVEN"
schedulingPeriodCRON: "* * * * * ?"
directory: "/tmp/input"
requirements:
- connectToPipeline:
node: compressData_0
relationship: con_ConnectNifiLocal_0
capability: ConnectToPipeline
- host:
node: Nifi_0
relationship: con_HostedOn_1
capability: host
OpenStack_0:
type: radon.nodes.VM.OpenStack
metadata:
x: "1275"
y: "570"
displayName: "OpenStack"
properties:
flavor: "c1.medium"
key_name: "jakovits_ldpc"
image: "21cd7486-a2a6-458c-9b05-9207c0b63819"
ssh_username: "ubuntu"
name: "sdfsdf"
network: "provider_64_net"
Nifi_0:
type: radon.nodes.nifi.Nifi
metadata:
x: "853"
y: "556"
displayName: "Nifi"
properties:
port: 8080
webinterface_public: true
component_version: "1.13.2"
requirements:
- host:
node: OpenStack_0
relationship: con_HostedOn_0
capability: host
relationship_templates:
con_HostedOn_2:
type: tosca.relationships.HostedOn
con_ConnectNifiLocal_3:
type: radon.relationships.datapipeline.ConnectNifiLocal
con_HostedOn_3:
type: tosca.relationships.HostedOn
con_HostedOn_0:
type: tosca.relationships.HostedOn
con_HostedOn_1:
type: tosca.relationships.HostedOn
con_ConnectNifiLocal_0:
type: radon.relationships.datapipeline.ConnectNifiLocal
con_ConnectNifiLocal_2:
type: radon.relationships.datapipeline.ConnectNifiLocal
con_ConnectNifiLocal_1:
type: radon.relationships.datapipeline.ConnectNifiLocal
con_HostedOn_4:
type: tosca.relationships.HostedOn
con_HostedOn_5:
type: tosca.relationships.HostedOn
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment