Merge pull request #85 from pjakovits/PolicyModifications

Policy modifications for Data Pipeline testing Agent

nodetypes/radon.nodes.datapipeline.source/ConsumeLocal/files/configure/configure.yml

@@ -20,6 +20,13 @@
     get_id: "processors[0].id"
     port: "{{ (lookup('file', '/tmp/nifi.json') | from_json) | default('8080', true) }}"
   tasks:
+
+    - name: Creates directory
+      file:
+        path: "{{directory}}"
+        state: directory
+        mode: 0777
+
     - name: Get list of available NiFi processors within the process group
       uri:
         url: "http://localhost:{{ port }}/nifi-api/process-groups/{{ pipeline_id }}/processors"

nodetypes/radon.nodes.nifi/NiFiDocker/files/create/create.yaml

@@ -21,11 +21,22 @@
         pull: true
         ports:
           - "{{port}}:8080"
+          - "9093:9093"
         volumes:
-          - /mnt:/mnt
+          - /tmp:/tmp
       become: true
 
     - name: Wait for port to become open on the host ( dont start checking for 40 seconds )
       wait_for:
         port: "{{port}}"
         delay: 40
+
+
+    - name: Write port variable to file
+      copy:
+         content: '{{ port | to_json }}'
+         dest: /tmp/nifi.json
+      delegate_to: 127.0.0.1
+      become: no
+
+

nodetypes/radon.nodes.testing/DataPipeline/files/create/create.yml

@@ -21,5 +21,5 @@
         ports:
           - "5000:5000"
         volumes:
-          - /mnt:/mnt
+          - /tmp:/tmp
       become: true

policytypes/radon.policies.testing/DataPipelineLoadTest/PolicyType.tosca

@@ -12,5 +12,15 @@ policy_types:
         type: integer
       test_duration:
         type: integer
+      lower_bound:
+        type: float
+        required: false
+      performance_metric:
+        type: string
+        required: false
+        default: "CPU_Load"
+      upper_bound:
+        type: float
+        required: false
       resource_location:
         type: string

policytypes/radon.policies.testing/DataPipelineLoadTest/README.md

@@ -11,5 +11,5 @@ This policy type represents a load test case for the Nifi based Data Pipelines.
 | Name | Required | Type | Constraint | Default Value | Description |
 |:---- |:-------- |:---- |:---------- |:------------- |:----------- |
 | `resources` | `true` | `string` |  |  | A single ZIP archive containing all resources for the test. | 
-| `user.properties` | `false` | `string` |   |   | Location and filename of a file with properties for the test plan (optional) |
+
 

servicetemplates/radon.blueprints.testing/DataPipelineAWSdemoSUT/ServiceTemplate.tosca

@@ -17,18 +17,18 @@ topology_template:
       type: string
       required: true
   node_templates:
-    ConsS3Bucket_1:
+    ConsS3Bucket_0:
       type: radon.nodes.datapipeline.source.ConsS3Bucket
       metadata:
-        x: "266"
-        y: "91"
+        x: "304"
+        y: "92"
         displayName: "ConsS3Bucket"
       properties:
         BucketName: "radon-jakovits"
         cred_file_path: { get_input: cred_file_path }
         schedulingStrategy: "EVENT_DRIVEN"
         schedulingPeriodCRON: "* * * * * ?"
-        name: "read_from_s3"
+        name: "readfroms3"
         Region: "eu-central-1"
       requirements:
         - connectToPipeline:
@@ -37,13 +37,13 @@ topology_template:
             capability: ConnectToPipeline
         - host:
             node: Nifi_0
-            relationship: con_HostedOn_0
+            relationship: con_HostedOn_2
             capability: host
     AwsPlatform_0:
       type: radon.nodes.aws.AwsPlatform
       metadata:
-        x: "786"
-        y: "939"
+        x: "612"
+        y: "680"
         displayName: "AwsPlatform"
       properties:
         name: "AWS"
@@ -51,26 +51,41 @@ topology_template:
     PubsS3Bucket_0:
       type: radon.nodes.datapipeline.destination.PubsS3Bucket
       metadata:
-        x: "1063"
-        y: "93"
+        x: "1031"
+        y: "81"
         displayName: "PubsS3Bucket"
       properties:
         BucketName: "radon-jakovits-output"
         cred_file_path: { get_input: cred_file_path }
         schedulingStrategy: "EVENT_DRIVEN"
         schedulingPeriodCRON: "* * * * * ?"
-        name: "send_to_s3"
+        name: "sendtos3"
         Region: "eu-central-1"
       requirements:
         - host:
             node: Nifi_0
             relationship: con_HostedOn_1
             capability: host
+    Nifi_0:
+      type: radon.nodes.nifi.Nifi
+      metadata:
+        x: "625"
+        y: "198"
+        displayName: "Nifi"
+      properties:
+        port: 8080
+        webinterface_public: true
+        component_version: "1.13.2"
+      requirements:
+        - host:
+            node: EC2_0
+            relationship: con_HostedOn_0
+            capability: host
     EC2_0:
       type: radon.nodes.VM.EC2
       metadata:
-        x: "694"
-        y: "788"
+        x: "604"
+        y: "446"
         displayName: "EC2"
       properties:
         image: "ami-0e0102e3ff768559b"
@@ -84,20 +99,6 @@ topology_template:
             node: AwsPlatform_0
             relationship: con_HostedOn_3
             capability: host
-    Nifi_0:
-      type: radon.nodes.nifi.Nifi
-      metadata:
-        x: "713"
-        y: "534"
-        displayName: "Nifi"
-      properties:
-        port: 8080
-        component_version: "1.13.1"
-      requirements:
-        - host:
-            node: EC2_0
-            relationship: con_HostedOn_2
-            capability: host
   relationship_templates:
     con_HostedOn_2:
       type: tosca.relationships.HostedOn
@@ -113,13 +114,17 @@ topology_template:
     - pipelineLoadTestPolicy:
         type: radon.policies.testing.DataPipelineLoadTest
         properties:
-          hostname: "https://radon-jakovits.s3.eu-central-1.amazonaws.com/"
+          velocity_per_minute: "80"
+          hostname: "radon-jakovits.s3.eu-central-1.amazonaws.com"
+          test_duration: "80"
+          lower_bound: "0.1"
           port: "8080"
-          resources: "resources.zip"
+          performance_metric: "CPU_Load"
           ti_blueprint: "radon.blueprints.testing.NiFiTIDocker"
-          user.properties: "None"
-          test_id: "loadtest556"
-        targets: [ ConsS3Bucket_1 ]
+          upper_bound: "0.7"
+          resource_location: "radon-ctt/resources.zip"
+          test_id: "datatest67"
+        targets: [ ConsS3Bucket_0 ]
   outputs:
     public_address:
       type: string

servicetemplates/radon.blueprints.testing/NiFiTIDocker/ServiceTemplate.tosca

@@ -13,6 +13,9 @@ topology_template:
     vpc_subnet_id:
       type: string
       required: true
+    s3_bucket_target:
+      type: string
+      required: true
     ssh_key_file:
       type: string
       required: false
@@ -69,9 +72,10 @@ topology_template:
         y: "140"
         displayName: "PubsS3Bucket"
       properties:
-        BucketName: "radon-jakovits"
+        BucketName: { get_input: s3_bucket_target }
         cred_file_path: { get_input: cred_file_path }
-        schedulingStrategy: "EVENT_DRIVENsend_to_s3"
+        schedulingStrategy: "EVENT_DRIVEN"
+        name: "send_tos3"
         schedulingPeriodCRON: "* * * * * ?"
         Region: "eu-central-1"
       requirements:
@@ -89,7 +93,7 @@ topology_template:
         schedulingStrategy: "EVENT_DRIVEN"
         name: "read_test_files"
         schedulingPeriodCRON: "* * * * * ?"
-        directory: "/mnt"
+        directory: "/tmp/nifi_agent"
       requirements:
         - connectToPipeline:
             node: PubsS3Bucket_0