From 2e16f323063fcd5689e7ab27daf7ae2db805ca0d Mon Sep 17 00:00:00 2001 From: Mansi Date: Thu, 1 Aug 2019 11:32:11 -0400 Subject: [PATCH 01/20] Initial commit, implemented post for credentials --- .../keys.yml} | 0 .../keys.yml | 5 ++ .../keys.yml | 6 ++ .../keys.yml | 1 + .../keys.yml | 12 ++++ .../keys.yml | 0 .../dummy/.rundb/rundb-255543641996716.json | 1 + .../dummy/PinFile | 60 +++++++++++++++++++ .../dummy/PinFile.json | 1 + .../dummy/README.rst | 15 +++++ .../dummy-new-44950f.inventory.inventory | 18 ++++++ .../dummy/inventories/dummy_cluster.inventory | 14 +++++ .../dummy/resources/dummy_cluster.output | 1 + .../dummy/resources/linchpin.latest | 1 + .../dummy/PinFile | 60 +++++++++++++++++++ .../dummy/PinFile.json | 7 +++ .../dummy/README.rst | 15 +++++ app/__init__.py | 45 +++++++++++++- app/config.yml | 8 ++- app/data_access_layer/UserBaseDB.py | 4 ++ app/data_access_layer/UserRestDB.py | 8 ++- 21 files changed, 277 insertions(+), 5 deletions(-) rename app/ /{44ad6c91-5191-43aa-8fa0-cd3fd2fc2e0d_dbtest/dummy/inventories/dummy_cluster.inventory2 => admin_1ab54521-c1a6-49ba-96b2-6b632bf68bd4/keys.yml} (100%) create mode 100644 app/ /admin_3044a4a8-6da4-461a-9800-d2207f2636da/keys.yml create mode 100644 app/ /admin_48a150fc-1160-4708-8b91-2d903c831f6c/keys.yml create mode 100644 app/ /admin_4f6b4d9f-9f2b-44b1-9377-da70988a9d0d/keys.yml create mode 100644 app/ /admin_727bc9c9-8f26-4738-b481-1eb5ca84c107/keys.yml create mode 100644 app/ /admin_892d2011-11af-4ee8-913a-e6600a052b08/keys.yml create mode 100644 app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/.rundb/rundb-255543641996716.json create mode 100644 app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/PinFile create mode 100644 app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/PinFile.json create mode 100644 app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/README.rst create mode 100644 app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/inventories/dummy-new-44950f.inventory.inventory create mode 100644 app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/inventories/dummy_cluster.inventory create mode 100644 app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/resources/dummy_cluster.output create mode 100644 app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/resources/linchpin.latest create mode 100644 app/ /tmp648b13a7-f048-45c5-8704-8bf8266e9d63_wk1/dummy/PinFile create mode 100644 app/ /tmp648b13a7-f048-45c5-8704-8bf8266e9d63_wk1/dummy/PinFile.json create mode 100644 app/ /tmp648b13a7-f048-45c5-8704-8bf8266e9d63_wk1/dummy/README.rst diff --git a/app/ /44ad6c91-5191-43aa-8fa0-cd3fd2fc2e0d_dbtest/dummy/inventories/dummy_cluster.inventory2 b/app/ /admin_1ab54521-c1a6-49ba-96b2-6b632bf68bd4/keys.yml similarity index 100% rename from app/ /44ad6c91-5191-43aa-8fa0-cd3fd2fc2e0d_dbtest/dummy/inventories/dummy_cluster.inventory2 rename to app/ /admin_1ab54521-c1a6-49ba-96b2-6b632bf68bd4/keys.yml diff --git a/app/ /admin_3044a4a8-6da4-461a-9800-d2207f2636da/keys.yml b/app/ /admin_3044a4a8-6da4-461a-9800-d2207f2636da/keys.yml new file mode 100644 index 0000000..e67e452 --- /dev/null +++ b/app/ /admin_3044a4a8-6da4-461a-9800-d2207f2636da/keys.yml @@ -0,0 +1,5 @@ +Flask>=1.0.3 +linchpin<=1.7.5 +flask-swagger-ui>=3.20.9 +flake8>=3.7.7 +ansible_vault>=1.2.0 diff --git a/app/ /admin_48a150fc-1160-4708-8b91-2d903c831f6c/keys.yml b/app/ /admin_48a150fc-1160-4708-8b91-2d903c831f6c/keys.yml new file mode 100644 index 0000000..9508520 --- /dev/null +++ b/app/ /admin_48a150fc-1160-4708-8b91-2d903c831f6c/keys.yml @@ -0,0 +1,6 @@ +$ANSIBLE_VAULT;1.1;AES256 +63613739623037633062303263633235613238323664623937306530383332383633323363626162 +3466306530636637386135326362313532326239386135320a643636643838336466353337393133 +38623033666331643361353663353835353237616635333462343739343062643935346238643261 +3436313636663238380a613133663764373465313731343734313739643964353937363063623536 +3737 diff --git a/app/ /admin_4f6b4d9f-9f2b-44b1-9377-da70988a9d0d/keys.yml b/app/ /admin_4f6b4d9f-9f2b-44b1-9377-da70988a9d0d/keys.yml new file mode 100644 index 0000000..de56dd9 --- /dev/null +++ b/app/ /admin_4f6b4d9f-9f2b-44b1-9377-da70988a9d0d/keys.yml @@ -0,0 +1 @@ +mansialli \ No newline at end of file diff --git a/app/ /admin_727bc9c9-8f26-4738-b481-1eb5ca84c107/keys.yml b/app/ /admin_727bc9c9-8f26-4738-b481-1eb5ca84c107/keys.yml new file mode 100644 index 0000000..420a1d4 --- /dev/null +++ b/app/ /admin_727bc9c9-8f26-4738-b481-1eb5ca84c107/keys.yml @@ -0,0 +1,12 @@ +$ANSIBLE_VAULT;1.1;AES256 +61383433356666363530643365356365333336323763366437343230333237636434383330386365 +3837396537636634326539663233663032613630666631350a376638313336393131636436306630 +38646530613335336332376635653566653236636364306433613330636461306231336138373062 +3338343762626364330a656363636261663666663861316132353031363631333630633331363861 +37613166363663326432663637336163636434313665343665373933373663633131356134333532 +38383632366239336435376361643937623238323838333062333139663530623436643739613936 +63373138636464343737666462666137373466363061656165666165306330303164663236333463 +39336330636630303665666334393836656633616332316134306532323264633337373339393661 +39633066333635313665633330646564656537313336333730656439626638643131623733366637 +35303062396562643261653838643161323934353364623332313838353139323130626539326362 +303230653565356266326638373866636637 diff --git a/app/ /admin_892d2011-11af-4ee8-913a-e6600a052b08/keys.yml b/app/ /admin_892d2011-11af-4ee8-913a-e6600a052b08/keys.yml new file mode 100644 index 0000000..e69de29 diff --git a/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/.rundb/rundb-255543641996716.json b/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/.rundb/rundb-255543641996716.json new file mode 100644 index 0000000..125648d --- /dev/null +++ b/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/.rundb/rundb-255543641996716.json @@ -0,0 +1 @@ +{"linchpin": {"1": {"action": "up", "targets": [{"dummy-new": {"1": {"rc": 0, "uhash": "44950f"}}, "dummy-topo": {"1": {"rc": 0, "uhash": "86864e"}}}]}}, "dummy-new": {"1": {"action": "up", "inputs": [{"topology_data": {"topology_name": "dummy_cluster", "resource_groups": [{"resource_group_name": "dummy", "resource_group_type": "dummy", "resource_definitions": [{"name": "web", "role": "dummy_node", "count": 3}, {"name": "test", "role": "dummy_node", "count": 1}]}]}}, {"layout_data": {"inventory_layout": {"inventory_file": "{{ workspace }}/inventories/dummy-new-{{ uhash }}.inventory", "vars": {"hostname": "__IP__"}, "hosts": [{"name": "example-node", "count": 3, "host_groups": ["example"]}, {"name": "test-node", "count": 1, "host_groups": ["test"]}], "host_groups": {"all": {"vars": {"ansible_user": "root", "ansible_private_key_file": "{{ lookup('env', 'TESTLP') | default('/tmp', true) }}/CSS/keystore/css-central"}}}}}}], "outputs": [{"resources": [{"changed": true, "hosts": ["web-0", "web-1", "web-2"], "dummy_file": "/tmp/dummy.hosts", "failed": false, "resource_type": "dummy_res"}, {"changed": true, "hosts": ["test"], "dummy_file": "/tmp/dummy.hosts", "failed": false, "resource_type": "dummy_res"}]}, {"inventory_path": ["/home/mankulka/development/linchpin/restylinchpin/restylinchpin/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/inventories/dummy-new-44950f.inventory.inventory"]}], "cfgs": [], "start": "1564154877.9498544", "end": "1564154882.8734448", "rc": 0, "uhash": "44950f"}}, "dummy-topo": {"1": {"action": "up", "inputs": [{"topology_data": {"topology_name": "dummy_cluster", "resource_groups": [{"resource_group_name": "dummy", "resource_group_type": "dummy", "resource_definitions": [{"name": "web", "role": "dummy_node", "count": 3}, {"name": "test", "role": "dummy_node", "count": 1}]}]}}, {"layout_data": {"inventory_layout": {"vars": {"hostname": "__IP__"}, "hosts": [{"name": "example-node", "count": 3, "host_groups": ["example"]}, {"name": "test-node", "count": 1, "host_groups": ["test"]}]}}}], "outputs": [{"resources": [{"changed": true, "hosts": ["web-0", "web-1", "web-2"], "dummy_file": "/tmp/dummy.hosts", "failed": false, "resource_type": "dummy_res"}, {"changed": true, "hosts": ["test"], "dummy_file": "/tmp/dummy.hosts", "failed": false, "resource_type": "dummy_res"}]}, {"inventory_path": ["/home/mankulka/development/linchpin/restylinchpin/restylinchpin/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/inventories/dummy_cluster.inventory"]}], "cfgs": [], "start": "1564154882.882087", "end": "1564154886.836086", "rc": 0, "uhash": "86864e"}}} \ No newline at end of file diff --git a/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/PinFile b/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/PinFile new file mode 100644 index 0000000..585ff72 --- /dev/null +++ b/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/PinFile @@ -0,0 +1,60 @@ +--- +dummy-new: + topology: + topology_name: "dummy_cluster" # topology name + resource_groups: + - resource_group_name: "dummy" + resource_group_type: "dummy" + resource_definitions: + - name: "{{ distro | default('') }}web" + role: "dummy_node" + count: 3 + - name: "{{ distro | default('') }}test" + role: "dummy_node" + count: 1 + layout: + inventory_layout: + inventory_file: "{% raw -%}{{ workspace }}/inventories/dummy-new-{{ uhash }}.inventory{%- endraw %}" + vars: + hostname: __IP__ + hosts: + example-node: + count: 3 + host_groups: + - example + test-node: + count: 1 + host_groups: + - test + host_groups: + all: + vars: + ansible_user: root + ansible_private_key_file: "{% raw -%}{{ lookup('env', 'TESTLP') | default('/tmp', true) }}/CSS/keystore/css-central{%- endraw %}" + +dummy-topo: + topology: + topology_name: "dummy_cluster" # topology name + resource_groups: + - resource_group_name: "dummy" + resource_group_type: "dummy" + resource_definitions: + - name: "{{ distro | default('') }}web" + role: "dummy_node" + count: 3 + - name: "{{ distro | default('') }}test" + role: "dummy_node" + count: 1 + layout: + inventory_layout: + vars: + hostname: __IP__ + hosts: + example-node: + count: 3 + host_groups: + - example + test-node: + count: 1 + host_groups: + - test diff --git a/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/PinFile.json b/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/PinFile.json new file mode 100644 index 0000000..a70d886 --- /dev/null +++ b/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/PinFile.json @@ -0,0 +1 @@ +{"dummy-prov2": {"topology": {"resource_groups": [{"resource_definitions": [{"count": 3, "name": "web", "role": "dummy_node"}], "resource_group_name": "dummy", "resource_group_type": "dummy"}], "topology_name": "dummy_cluster"}, "layout": {"inventory_layout": {"hosts": {"example-node": {"count": 3, "host_groups": ["example"]}}, "vars": {"hostname": "__IP__"}}}}} \ No newline at end of file diff --git a/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/README.rst b/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/README.rst new file mode 100644 index 0000000..e98aafa --- /dev/null +++ b/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/README.rst @@ -0,0 +1,15 @@ +Dummy +====== + +This workspace contains the basic tasks for provisioning and teardown of dummy clusters. + +As the name implies, dummy clusters do not provision any resources, but do run through +the provisioning steps for easy testing of linchpin features + +dummy-new +======== +Provisions a new set of dummy resources and creates a layout to go with them. The name of the inventory file is dynamically generated based on the uhash + +dummy-topo +======== +Another set of dummy resources with a slightly simpler layout diff --git a/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/inventories/dummy-new-44950f.inventory.inventory b/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/inventories/dummy-new-44950f.inventory.inventory new file mode 100644 index 0000000..2186d29 --- /dev/null +++ b/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/inventories/dummy-new-44950f.inventory.inventory @@ -0,0 +1,18 @@ +[all:vars] +ansible_user = root +ansible_private_key_file = {{ lookup('env', 'TESTLP') | default('/tmp', true) }}/CSS/keystore/css-central + +[test] +test hostname=test + +[example] +web-0 hostname=web-0 +web-1 hostname=web-1 +web-2 hostname=web-2 + +[all] +test hostname=test +web-2 hostname=web-2 +web-1 hostname=web-1 +web-0 hostname=web-0 + diff --git a/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/inventories/dummy_cluster.inventory b/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/inventories/dummy_cluster.inventory new file mode 100644 index 0000000..c496581 --- /dev/null +++ b/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/inventories/dummy_cluster.inventory @@ -0,0 +1,14 @@ +[test] +test hostname=test + +[example] +web-0 hostname=web-0 +web-1 hostname=web-1 +web-2 hostname=web-2 + +[all] +test hostname=test +web-2 hostname=web-2 +web-1 hostname=web-1 +web-0 hostname=web-0 + diff --git a/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/resources/dummy_cluster.output b/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/resources/dummy_cluster.output new file mode 100644 index 0000000..c2626a2 --- /dev/null +++ b/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/resources/dummy_cluster.output @@ -0,0 +1 @@ +[{"changed": true, "hosts": ["web-0", "web-1", "web-2"], "dummy_file": "/tmp/dummy.hosts", "failed": false, "resource_type": "dummy_res"}, {"changed": true, "hosts": ["test"], "dummy_file": "/tmp/dummy.hosts", "failed": false, "resource_type": "dummy_res"}] diff --git a/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/resources/linchpin.latest b/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/resources/linchpin.latest new file mode 100644 index 0000000..a5f01d2 --- /dev/null +++ b/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/resources/linchpin.latest @@ -0,0 +1 @@ +{"1": {"action": "up", "targets": [{"dummy-new": {"1": {"rc": 0, "uhash": "44950f"}, "outputs": {"resources": [{"changed": true, "hosts": ["web-0", "web-1", "web-2"], "dummy_file": "/tmp/dummy.hosts", "failed": false, "resource_type": "dummy_res"}, {"changed": true, "hosts": ["test"], "dummy_file": "/tmp/dummy.hosts", "failed": false, "resource_type": "dummy_res"}], "inventory_path": ["/home/mankulka/development/linchpin/restylinchpin/restylinchpin/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/inventories/dummy-new-44950f.inventory.inventory"]}, "inputs": {"topology_data": {"topology_name": "dummy_cluster", "resource_groups": [{"resource_group_name": "dummy", "resource_group_type": "dummy", "resource_definitions": [{"name": "web", "role": "dummy_node", "count": 3}, {"name": "test", "role": "dummy_node", "count": 1}]}]}, "layout_data": {"inventory_layout": {"inventory_file": "{{ workspace }}/inventories/dummy-new-{{ uhash }}.inventory", "vars": {"hostname": "__IP__"}, "hosts": [{"name": "example-node", "count": 3, "host_groups": ["example"]}, {"name": "test-node", "count": 1, "host_groups": ["test"]}], "host_groups": {"all": {"vars": {"ansible_user": "root", "ansible_private_key_file": "{{ lookup('env', 'TESTLP') | default('/tmp', true) }}/CSS/keystore/css-central"}}}}}}}, "dummy-topo": {"1": {"rc": 0, "uhash": "86864e"}, "outputs": {"resources": [{"changed": true, "hosts": ["web-0", "web-1", "web-2"], "dummy_file": "/tmp/dummy.hosts", "failed": false, "resource_type": "dummy_res"}, {"changed": true, "hosts": ["test"], "dummy_file": "/tmp/dummy.hosts", "failed": false, "resource_type": "dummy_res"}], "inventory_path": ["/home/mankulka/development/linchpin/restylinchpin/restylinchpin/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/inventories/dummy_cluster.inventory"]}, "inputs": {"topology_data": {"topology_name": "dummy_cluster", "resource_groups": [{"resource_group_name": "dummy", "resource_group_type": "dummy", "resource_definitions": [{"name": "web", "role": "dummy_node", "count": 3}, {"name": "test", "role": "dummy_node", "count": 1}]}]}, "layout_data": {"inventory_layout": {"vars": {"hostname": "__IP__"}, "hosts": [{"name": "example-node", "count": 3, "host_groups": ["example"]}, {"name": "test-node", "count": 1, "host_groups": ["test"]}]}}}}}]}} \ No newline at end of file diff --git a/app/ /tmp648b13a7-f048-45c5-8704-8bf8266e9d63_wk1/dummy/PinFile b/app/ /tmp648b13a7-f048-45c5-8704-8bf8266e9d63_wk1/dummy/PinFile new file mode 100644 index 0000000..585ff72 --- /dev/null +++ b/app/ /tmp648b13a7-f048-45c5-8704-8bf8266e9d63_wk1/dummy/PinFile @@ -0,0 +1,60 @@ +--- +dummy-new: + topology: + topology_name: "dummy_cluster" # topology name + resource_groups: + - resource_group_name: "dummy" + resource_group_type: "dummy" + resource_definitions: + - name: "{{ distro | default('') }}web" + role: "dummy_node" + count: 3 + - name: "{{ distro | default('') }}test" + role: "dummy_node" + count: 1 + layout: + inventory_layout: + inventory_file: "{% raw -%}{{ workspace }}/inventories/dummy-new-{{ uhash }}.inventory{%- endraw %}" + vars: + hostname: __IP__ + hosts: + example-node: + count: 3 + host_groups: + - example + test-node: + count: 1 + host_groups: + - test + host_groups: + all: + vars: + ansible_user: root + ansible_private_key_file: "{% raw -%}{{ lookup('env', 'TESTLP') | default('/tmp', true) }}/CSS/keystore/css-central{%- endraw %}" + +dummy-topo: + topology: + topology_name: "dummy_cluster" # topology name + resource_groups: + - resource_group_name: "dummy" + resource_group_type: "dummy" + resource_definitions: + - name: "{{ distro | default('') }}web" + role: "dummy_node" + count: 3 + - name: "{{ distro | default('') }}test" + role: "dummy_node" + count: 1 + layout: + inventory_layout: + vars: + hostname: __IP__ + hosts: + example-node: + count: 3 + host_groups: + - example + test-node: + count: 1 + host_groups: + - test diff --git a/app/ /tmp648b13a7-f048-45c5-8704-8bf8266e9d63_wk1/dummy/PinFile.json b/app/ /tmp648b13a7-f048-45c5-8704-8bf8266e9d63_wk1/dummy/PinFile.json new file mode 100644 index 0000000..c3c942b --- /dev/null +++ b/app/ /tmp648b13a7-f048-45c5-8704-8bf8266e9d63_wk1/dummy/PinFile.json @@ -0,0 +1,7 @@ +{ + "dummy": { + "topology": "dummy.json", + "layout": "dummy.json" + } +} + diff --git a/app/ /tmp648b13a7-f048-45c5-8704-8bf8266e9d63_wk1/dummy/README.rst b/app/ /tmp648b13a7-f048-45c5-8704-8bf8266e9d63_wk1/dummy/README.rst new file mode 100644 index 0000000..e98aafa --- /dev/null +++ b/app/ /tmp648b13a7-f048-45c5-8704-8bf8266e9d63_wk1/dummy/README.rst @@ -0,0 +1,15 @@ +Dummy +====== + +This workspace contains the basic tasks for provisioning and teardown of dummy clusters. + +As the name implies, dummy clusters do not provision any resources, but do run through +the provisioning steps for easy testing of linchpin features + +dummy-new +======== +Provisions a new set of dummy resources and creates a layout to go with them. The name of the inventory file is dynamically generated based on the uhash + +dummy-topo +======== +Another set of dummy resources with a slightly simpler layout diff --git a/app/__init__.py b/app/__init__.py index 8e4cdca..f201d7b 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -7,6 +7,7 @@ import shutil import logging import subprocess +from ansible_vault import Vault from app.response_messages import response, errors from logging.handlers import RotatingFileHandler from flask import Flask, jsonify, request, Response, abort, make_response @@ -30,7 +31,7 @@ # loads defaults when config.yml does not exists or has been removed -WORKSPACE_DIR = config.get('workspace_path', '/tmp') +WORKSPACE_DIR = config.get('workspace_path', '/') LOGGER_FILE = config.get('logger_file_name', 'restylinchpin.log') DB_PATH = config.get('db_path', 'db.json') INVENTORY_PATH = config.get('inventory_path', '/dummy/inventories/*') @@ -41,6 +42,7 @@ ADMIN_USERNAME = config.get('admin_username', 'admin') ADMIN_PASSWORD = config.get('admin_password', 'password') ADMIN_EMAIL = config.get('admin_email', 'email') +CREDS_PATH = config.get('creds_path', '/') # URL for exposing Swagger UI (without trailing '/') SWAGGER_URL = '/api/docs' @@ -746,6 +748,47 @@ def get_linchpin_inventory(current_user, identity) -> Response: return jsonify(status=errors.ERROR_STATUS, message=str(e)) +@app.route('/api/v1.0/credentials', methods=['POST']) +@auth_required +def upload_credentials(current_user) -> Response: + db_con = get_connection_users(DB_PATH) + try: + file_name = request.form['file_name'] + encrypted = request.form['encrypted'] + if current_user['creds_folder'] is None: + creds_folder = current_user['username'] + "_" + str(uuid.uuid4()) + db_con.db_update_creds_folder(current_user['username'], + creds_folder) + os.makedirs(WORKSPACE_PATH + CREDS_PATH + creds_folder) + else: + creds_folder = current_user['creds_folder'] + if request.files: + file = request.files["file"] + file_read = file.read() + else: + file_read = request.form["file"] + if encrypted == "True": + if request.files: + with open(WORKSPACE_PATH + CREDS_PATH + creds_folder + "/" + file_name + ".yml", 'wb') as yaml_file: + print(os.getcwd()) + yaml_file.write(file_read) + else: + with open(WORKSPACE_PATH + CREDS_PATH + creds_folder + "/" + file_name + ".yml", 'w') as yaml_file: + print(os.getcwd()) + yaml_file.write(file_read) + else: + vault_pass = request.form['vault_pass'] + vault = Vault(vault_pass) + vault.dump(file_read, open(WORKSPACE_PATH + CREDS_PATH + creds_folder + "/" + file_name + ".yml", 'wb')) + return jsonify(message="Credentials uploaded successfully") + except (KeyError, ValueError, TypeError): + return jsonify(status=errors.ERROR_STATUS, + message=errors.KEY_ERROR) + except Exception as e: + app.logger.error(e) + return jsonify(status=errors.ERROR_STATUS, message=str(e)) + + if __name__ == "__main__": create_admin_user(DB_PATH, ADMIN_USERNAME, ADMIN_PASSWORD, ADMIN_EMAIL) diff --git a/app/config.yml b/app/config.yml index 16354a9..106721d 100644 --- a/app/config.yml +++ b/app/config.yml @@ -1,6 +1,6 @@ # place where all the created workspaces are stored # workspace_path --> place where are the workspaces are created and stored -workspace_path: /tmp +workspace_path: / # name or path of logger file logger_file_name: restylinchpin.log # path to workspace tinydb source file @@ -11,9 +11,11 @@ inventory_path: /dummy/inventories/* linchpin_latest_file_path: /dummy/resources/linchpin.latest # path to Pinfile for dummy workspace pinfile_json_path: /dummy/PinFile.json -#details for admin user login #file name for linchpin.latest file in resources linchpin_latest_name: linchpin.latest +#details for admin user login admin_username: admin admin_password: password -admin_email: admin@xyz \ No newline at end of file +admin_email: admin@xyz +# path to creds folder +creds_path: / \ No newline at end of file diff --git a/app/data_access_layer/UserBaseDB.py b/app/data_access_layer/UserBaseDB.py index 0216bce..e2a86d1 100644 --- a/app/data_access_layer/UserBaseDB.py +++ b/app/data_access_layer/UserBaseDB.py @@ -42,3 +42,7 @@ def db_reset_api_key(self, username, new_api_key): @abstractmethod def db_update(self, username, updated_username, password_hash, email): pass + + @abstractmethod + def db_update_creds_folder(self, username, creds_folder): + pass diff --git a/app/data_access_layer/UserRestDB.py b/app/data_access_layer/UserRestDB.py index ba8300c..a52df56 100644 --- a/app/data_access_layer/UserRestDB.py +++ b/app/data_access_layer/UserRestDB.py @@ -24,9 +24,11 @@ def db_insert(self, username, password_hash, api_key_hash, admin user :param email: User email """ + creds_folder = None self.table.insert({'username': username, 'password': password_hash, 'api_key': api_key_hash, - 'email': email, 'admin': admin}) + 'email': email, 'admin': admin, + 'creds_folder': creds_folder}) def db_search_name(self, username) -> List[Dict]: """ @@ -113,3 +115,7 @@ def db_update(self, username, updated_username, password_hash, self.table.update({'username': updated_username, 'password': password_hash, 'email': email}, user.username == username) + + def db_update_creds_folder(self, username, creds_folder): + user = Query() + self.table.update({'creds_folder': creds_folder}, user.username == username) From 9ef5f9e684db56c4050725ecbea3076ec9712b93 Mon Sep 17 00:00:00 2001 From: Mansi Date: Thu, 1 Aug 2019 16:02:55 -0400 Subject: [PATCH 02/20] Added authentication, get routes for credentials --- app/__init__.py | 25 +++++++++++++++++++++---- app/response_messages/response.py | 3 +++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index f201d7b..44c31bc 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -753,6 +753,9 @@ def get_linchpin_inventory(current_user, identity) -> Response: def upload_credentials(current_user) -> Response: db_con = get_connection_users(DB_PATH) try: + user = db_con.db_search_name(current_user['username']) + if not user: + return jsonify(message=response.USER_NOT_FOUND) file_name = request.form['file_name'] encrypted = request.form['encrypted'] if current_user['creds_folder'] is None: @@ -767,20 +770,18 @@ def upload_credentials(current_user) -> Response: file_read = file.read() else: file_read = request.form["file"] - if encrypted == "True": + if encrypted.lower() in ("true", "t"): if request.files: with open(WORKSPACE_PATH + CREDS_PATH + creds_folder + "/" + file_name + ".yml", 'wb') as yaml_file: - print(os.getcwd()) yaml_file.write(file_read) else: with open(WORKSPACE_PATH + CREDS_PATH + creds_folder + "/" + file_name + ".yml", 'w') as yaml_file: - print(os.getcwd()) yaml_file.write(file_read) else: vault_pass = request.form['vault_pass'] vault = Vault(vault_pass) vault.dump(file_read, open(WORKSPACE_PATH + CREDS_PATH + creds_folder + "/" + file_name + ".yml", 'wb')) - return jsonify(message="Credentials uploaded successfully") + return jsonify(message=response.CREDENTIALS_UPLOADED) except (KeyError, ValueError, TypeError): return jsonify(status=errors.ERROR_STATUS, message=errors.KEY_ERROR) @@ -789,6 +790,22 @@ def upload_credentials(current_user) -> Response: return jsonify(status=errors.ERROR_STATUS, message=str(e)) +@app.route('/api/v1.0/credentials/', methods=['GET']) +@auth_required +def get_credentials(current_user, file_name) -> Response: + db_con = get_connection_users(DB_PATH) + user = db_con.db_search_name(current_user['username']) + if not user: + return jsonify(message=response.USER_NOT_FOUND) + if not os.listdir(WORKSPACE_PATH + CREDS_PATH + current_user['creds_folder']). \ + __contains__(file_name): + return jsonify(message=response.CREDENTIALS_FILE_NOT_FOUND) + with open(WORKSPACE_PATH + CREDS_PATH + current_user['creds_folder'] + + "/" + file_name, 'r') as data: + credentials = data.read().replace('\n', ' ') + return jsonify(credentials=credentials) + + if __name__ == "__main__": create_admin_user(DB_PATH, ADMIN_USERNAME, ADMIN_PASSWORD, ADMIN_EMAIL) diff --git a/app/response_messages/response.py b/app/response_messages/response.py index 3d8ecb8..dabf779 100644 --- a/app/response_messages/response.py +++ b/app/response_messages/response.py @@ -3,6 +3,7 @@ PROVISION_SUCCESS = "Workspace provisioned successfully" DESTROY_SUCCESS = "Workspace/resources destroyed successfully" NOT_FOUND = "Workspace does not exist" +NOT_FOUND_USER = "User account does not exist" DUPLICATE_WORKSPACE = "Workspace with same name found," \ "try again by renaming" EMPTY_WORKSPACE = "Only public repositories can be "\ @@ -19,6 +20,7 @@ LINCHPIN_LATEST_NOT_FOUND = "linchpin.latest not found. " \ "Please check that it exists"\ " or specify linchpin_latest_path in request" +CREDENTIALS_FILE_NOT_FOUND = "No credential file found with this name" DESTROY_STATUS_SUCCESS = "Destroyed" DESTROY_FAILED = "FAILED destroy" STATUS_OK = "200 OK" @@ -36,3 +38,4 @@ "the format route?api_key=value" MISSING_USERNAME = "please provide the username in request route in "\ "the format route?username=value" +CREDENTIALS_UPLOADED = "Credentials uploaded successfully" From ebfc0d5c09a2c0c498a68c7a268de78bde4c8444 Mon Sep 17 00:00:00 2001 From: Mansi Date: Thu, 1 Aug 2019 17:08:49 -0400 Subject: [PATCH 03/20] Added update and delete routes for credentials --- app/__init__.py | 106 +++++++++++++++++++++++++----- app/response_messages/response.py | 2 + 2 files changed, 92 insertions(+), 16 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 44c31bc..7b74974 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -758,13 +758,16 @@ def upload_credentials(current_user) -> Response: return jsonify(message=response.USER_NOT_FOUND) file_name = request.form['file_name'] encrypted = request.form['encrypted'] - if current_user['creds_folder'] is None: - creds_folder = current_user['username'] + "_" + str(uuid.uuid4()) - db_con.db_update_creds_folder(current_user['username'], - creds_folder) - os.makedirs(WORKSPACE_PATH + CREDS_PATH + creds_folder) + if request.form["creds_folder_name"]: + creds_folder = request.form["creds_folder_name"] else: - creds_folder = current_user['creds_folder'] + if current_user['creds_folder'] is None: + creds_folder = current_user['username'] + "_" + str(uuid.uuid4()) + db_con.db_update_creds_folder(current_user['username'], + creds_folder) + os.makedirs(WORKSPACE_PATH + CREDS_PATH + creds_folder) + else: + creds_folder = current_user['creds_folder'] if request.files: file = request.files["file"] file_read = file.read() @@ -794,16 +797,87 @@ def upload_credentials(current_user) -> Response: @auth_required def get_credentials(current_user, file_name) -> Response: db_con = get_connection_users(DB_PATH) - user = db_con.db_search_name(current_user['username']) - if not user: - return jsonify(message=response.USER_NOT_FOUND) - if not os.listdir(WORKSPACE_PATH + CREDS_PATH + current_user['creds_folder']). \ - __contains__(file_name): - return jsonify(message=response.CREDENTIALS_FILE_NOT_FOUND) - with open(WORKSPACE_PATH + CREDS_PATH + current_user['creds_folder'] + - "/" + file_name, 'r') as data: - credentials = data.read().replace('\n', ' ') - return jsonify(credentials=credentials) + try: + user = db_con.db_search_name(current_user['username']) + if not user: + return jsonify(message=response.USER_NOT_FOUND) + if not os.listdir(WORKSPACE_PATH + CREDS_PATH + current_user['creds_folder']). \ + __contains__(file_name): + return jsonify(message=response.CREDENTIALS_FILE_NOT_FOUND) + with open(WORKSPACE_PATH + CREDS_PATH + current_user['creds_folder'] + + "/" + file_name, 'r') as data: + credentials = data.read().replace('\n', ' ') + return jsonify(credentials=credentials) + except (KeyError, ValueError, TypeError): + return jsonify(status=errors.ERROR_STATUS, + message=errors.KEY_ERROR) + except Exception as e: + app.logger.error(e) + return jsonify(status=errors.ERROR_STATUS, message=str(e)) + + +@app.route('/api/v1.0/credentials/', methods=['PUT']) +@auth_required +def update_credentials(current_user, file_name) -> Response: + db_con = get_connection_users(DB_PATH) + try: + user = db_con.db_search_name(current_user['username']) + if not user: + return jsonify(message=response.USER_NOT_FOUND) + if 'creds_folder_name' not in request.form: + creds_folder = current_user['creds_folder'] + else: + creds_folder = request.form['creds_folder_name'] + if not os.listdir(WORKSPACE_PATH + CREDS_PATH + creds_folder). \ + __contains__(file_name): + return jsonify(message=response.CREDENTIALS_FILE_NOT_FOUND) + if request.files: + file = request.files["file"] + file_read = file.read() + else: + file_read = request.form["file"] + encrypted = request.form['encrypted'] + if encrypted.lower() in ("true", "t"): + if request.files: + with open(WORKSPACE_PATH + CREDS_PATH + creds_folder + "/" + file_name, 'wb') as yaml_file: + yaml_file.write(file_read) + else: + with open(WORKSPACE_PATH + CREDS_PATH + creds_folder + "/" + file_name, 'w') as yaml_file: + yaml_file.write(file_read) + else: + vault_pass = request.form['vault_pass'] + vault = Vault(vault_pass) + vault.dump(file_read, open(WORKSPACE_PATH + CREDS_PATH + creds_folder + "/" + file_name, 'wb')) + return jsonify(message=response.CREDENTIALS_UPDATED) + except Exception as e: + app.logger.error(e) + return jsonify(status=errors.ERROR_STATUS, message=str(e)) + + +@app.route('/api/v1.0/credentials/', methods=['DELETE']) +@auth_required +def delete_credentials(current_user, file_name) -> Response: + db_con = get_connection_users(DB_PATH) + try: + user = db_con.db_search_name(current_user['username']) + if not user: + return jsonify(message=response.USER_NOT_FOUND) + if 'creds_folder_name' not in request.form: + creds_folder = current_user['creds_folder'] + else: + creds_folder = request.form['creds_folder_name'] + for w in os.listdir(WORKSPACE_PATH + CREDS_PATH + creds_folder): + print(w) + if w == file_name: + print("here") + os.remove(WORKSPACE_PATH + CREDS_PATH + creds_folder + "/" + w) + print("deleted") + return jsonify(status=response.CREDENTIALS_DELETED, + mimetype='application/json') + return jsonify(status=response.CREDENTIALS_FILE_NOT_FOUND) + except Exception as e: + app.logger.error(e) + return jsonify(status=errors.ERROR_STATUS, message=str(e)) if __name__ == "__main__": diff --git a/app/response_messages/response.py b/app/response_messages/response.py index dabf779..7da667e 100644 --- a/app/response_messages/response.py +++ b/app/response_messages/response.py @@ -39,3 +39,5 @@ MISSING_USERNAME = "please provide the username in request route in "\ "the format route?username=value" CREDENTIALS_UPLOADED = "Credentials uploaded successfully" +CREDENTIALS_UPDATED = "Credentials updated sccessfully" +CREDENTIALS_DELETED = "Credentials deleted successfully" From 28dd5319e049a4532c5f6c66f4aa47aba44f551e Mon Sep 17 00:00:00 2001 From: Mansi Date: Fri, 2 Aug 2019 18:58:23 -0400 Subject: [PATCH 04/20] Removed unnecessary files, added requirements, fixed conflicts --- .../keys.yml | 0 .../keys.yml | 5 - .../keys.yml | 6 - .../keys.yml | 1 - .../keys.yml | 12 -- .../keys.yml | 0 .../dummy/.rundb/rundb-255543641996716.json | 1 - .../dummy/PinFile | 60 --------- .../dummy/PinFile.json | 1 - .../dummy/README.rst | 15 --- .../dummy-new-44950f.inventory.inventory | 18 --- .../dummy/inventories/dummy_cluster.inventory | 14 -- .../dummy/resources/dummy_cluster.output | 1 - .../dummy/resources/linchpin.latest | 1 - .../dummy/PinFile | 60 --------- .../dummy/PinFile.json | 7 - .../dummy/README.rst | 15 --- app/__init__.py | 127 ++++++++++++------ app/config.yml | 4 +- app/data_access_layer/UserRestDB.py | 7 +- requirements.txt | 1 + 21 files changed, 91 insertions(+), 265 deletions(-) delete mode 100644 app/ /admin_1ab54521-c1a6-49ba-96b2-6b632bf68bd4/keys.yml delete mode 100644 app/ /admin_3044a4a8-6da4-461a-9800-d2207f2636da/keys.yml delete mode 100644 app/ /admin_48a150fc-1160-4708-8b91-2d903c831f6c/keys.yml delete mode 100644 app/ /admin_4f6b4d9f-9f2b-44b1-9377-da70988a9d0d/keys.yml delete mode 100644 app/ /admin_727bc9c9-8f26-4738-b481-1eb5ca84c107/keys.yml delete mode 100644 app/ /admin_892d2011-11af-4ee8-913a-e6600a052b08/keys.yml delete mode 100644 app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/.rundb/rundb-255543641996716.json delete mode 100644 app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/PinFile delete mode 100644 app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/PinFile.json delete mode 100644 app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/README.rst delete mode 100644 app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/inventories/dummy-new-44950f.inventory.inventory delete mode 100644 app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/inventories/dummy_cluster.inventory delete mode 100644 app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/resources/dummy_cluster.output delete mode 100644 app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/resources/linchpin.latest delete mode 100644 app/ /tmp648b13a7-f048-45c5-8704-8bf8266e9d63_wk1/dummy/PinFile delete mode 100644 app/ /tmp648b13a7-f048-45c5-8704-8bf8266e9d63_wk1/dummy/PinFile.json delete mode 100644 app/ /tmp648b13a7-f048-45c5-8704-8bf8266e9d63_wk1/dummy/README.rst diff --git a/app/ /admin_1ab54521-c1a6-49ba-96b2-6b632bf68bd4/keys.yml b/app/ /admin_1ab54521-c1a6-49ba-96b2-6b632bf68bd4/keys.yml deleted file mode 100644 index e69de29..0000000 diff --git a/app/ /admin_3044a4a8-6da4-461a-9800-d2207f2636da/keys.yml b/app/ /admin_3044a4a8-6da4-461a-9800-d2207f2636da/keys.yml deleted file mode 100644 index e67e452..0000000 --- a/app/ /admin_3044a4a8-6da4-461a-9800-d2207f2636da/keys.yml +++ /dev/null @@ -1,5 +0,0 @@ -Flask>=1.0.3 -linchpin<=1.7.5 -flask-swagger-ui>=3.20.9 -flake8>=3.7.7 -ansible_vault>=1.2.0 diff --git a/app/ /admin_48a150fc-1160-4708-8b91-2d903c831f6c/keys.yml b/app/ /admin_48a150fc-1160-4708-8b91-2d903c831f6c/keys.yml deleted file mode 100644 index 9508520..0000000 --- a/app/ /admin_48a150fc-1160-4708-8b91-2d903c831f6c/keys.yml +++ /dev/null @@ -1,6 +0,0 @@ -$ANSIBLE_VAULT;1.1;AES256 -63613739623037633062303263633235613238323664623937306530383332383633323363626162 -3466306530636637386135326362313532326239386135320a643636643838336466353337393133 -38623033666331643361353663353835353237616635333462343739343062643935346238643261 -3436313636663238380a613133663764373465313731343734313739643964353937363063623536 -3737 diff --git a/app/ /admin_4f6b4d9f-9f2b-44b1-9377-da70988a9d0d/keys.yml b/app/ /admin_4f6b4d9f-9f2b-44b1-9377-da70988a9d0d/keys.yml deleted file mode 100644 index de56dd9..0000000 --- a/app/ /admin_4f6b4d9f-9f2b-44b1-9377-da70988a9d0d/keys.yml +++ /dev/null @@ -1 +0,0 @@ -mansialli \ No newline at end of file diff --git a/app/ /admin_727bc9c9-8f26-4738-b481-1eb5ca84c107/keys.yml b/app/ /admin_727bc9c9-8f26-4738-b481-1eb5ca84c107/keys.yml deleted file mode 100644 index 420a1d4..0000000 --- a/app/ /admin_727bc9c9-8f26-4738-b481-1eb5ca84c107/keys.yml +++ /dev/null @@ -1,12 +0,0 @@ -$ANSIBLE_VAULT;1.1;AES256 -61383433356666363530643365356365333336323763366437343230333237636434383330386365 -3837396537636634326539663233663032613630666631350a376638313336393131636436306630 -38646530613335336332376635653566653236636364306433613330636461306231336138373062 -3338343762626364330a656363636261663666663861316132353031363631333630633331363861 -37613166363663326432663637336163636434313665343665373933373663633131356134333532 -38383632366239336435376361643937623238323838333062333139663530623436643739613936 -63373138636464343737666462666137373466363061656165666165306330303164663236333463 -39336330636630303665666334393836656633616332316134306532323264633337373339393661 -39633066333635313665633330646564656537313336333730656439626638643131623733366637 -35303062396562643261653838643161323934353364623332313838353139323130626539326362 -303230653565356266326638373866636637 diff --git a/app/ /admin_892d2011-11af-4ee8-913a-e6600a052b08/keys.yml b/app/ /admin_892d2011-11af-4ee8-913a-e6600a052b08/keys.yml deleted file mode 100644 index e69de29..0000000 diff --git a/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/.rundb/rundb-255543641996716.json b/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/.rundb/rundb-255543641996716.json deleted file mode 100644 index 125648d..0000000 --- a/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/.rundb/rundb-255543641996716.json +++ /dev/null @@ -1 +0,0 @@ -{"linchpin": {"1": {"action": "up", "targets": [{"dummy-new": {"1": {"rc": 0, "uhash": "44950f"}}, "dummy-topo": {"1": {"rc": 0, "uhash": "86864e"}}}]}}, "dummy-new": {"1": {"action": "up", "inputs": [{"topology_data": {"topology_name": "dummy_cluster", "resource_groups": [{"resource_group_name": "dummy", "resource_group_type": "dummy", "resource_definitions": [{"name": "web", "role": "dummy_node", "count": 3}, {"name": "test", "role": "dummy_node", "count": 1}]}]}}, {"layout_data": {"inventory_layout": {"inventory_file": "{{ workspace }}/inventories/dummy-new-{{ uhash }}.inventory", "vars": {"hostname": "__IP__"}, "hosts": [{"name": "example-node", "count": 3, "host_groups": ["example"]}, {"name": "test-node", "count": 1, "host_groups": ["test"]}], "host_groups": {"all": {"vars": {"ansible_user": "root", "ansible_private_key_file": "{{ lookup('env', 'TESTLP') | default('/tmp', true) }}/CSS/keystore/css-central"}}}}}}], "outputs": [{"resources": [{"changed": true, "hosts": ["web-0", "web-1", "web-2"], "dummy_file": "/tmp/dummy.hosts", "failed": false, "resource_type": "dummy_res"}, {"changed": true, "hosts": ["test"], "dummy_file": "/tmp/dummy.hosts", "failed": false, "resource_type": "dummy_res"}]}, {"inventory_path": ["/home/mankulka/development/linchpin/restylinchpin/restylinchpin/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/inventories/dummy-new-44950f.inventory.inventory"]}], "cfgs": [], "start": "1564154877.9498544", "end": "1564154882.8734448", "rc": 0, "uhash": "44950f"}}, "dummy-topo": {"1": {"action": "up", "inputs": [{"topology_data": {"topology_name": "dummy_cluster", "resource_groups": [{"resource_group_name": "dummy", "resource_group_type": "dummy", "resource_definitions": [{"name": "web", "role": "dummy_node", "count": 3}, {"name": "test", "role": "dummy_node", "count": 1}]}]}}, {"layout_data": {"inventory_layout": {"vars": {"hostname": "__IP__"}, "hosts": [{"name": "example-node", "count": 3, "host_groups": ["example"]}, {"name": "test-node", "count": 1, "host_groups": ["test"]}]}}}], "outputs": [{"resources": [{"changed": true, "hosts": ["web-0", "web-1", "web-2"], "dummy_file": "/tmp/dummy.hosts", "failed": false, "resource_type": "dummy_res"}, {"changed": true, "hosts": ["test"], "dummy_file": "/tmp/dummy.hosts", "failed": false, "resource_type": "dummy_res"}]}, {"inventory_path": ["/home/mankulka/development/linchpin/restylinchpin/restylinchpin/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/inventories/dummy_cluster.inventory"]}], "cfgs": [], "start": "1564154882.882087", "end": "1564154886.836086", "rc": 0, "uhash": "86864e"}}} \ No newline at end of file diff --git a/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/PinFile b/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/PinFile deleted file mode 100644 index 585ff72..0000000 --- a/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/PinFile +++ /dev/null @@ -1,60 +0,0 @@ ---- -dummy-new: - topology: - topology_name: "dummy_cluster" # topology name - resource_groups: - - resource_group_name: "dummy" - resource_group_type: "dummy" - resource_definitions: - - name: "{{ distro | default('') }}web" - role: "dummy_node" - count: 3 - - name: "{{ distro | default('') }}test" - role: "dummy_node" - count: 1 - layout: - inventory_layout: - inventory_file: "{% raw -%}{{ workspace }}/inventories/dummy-new-{{ uhash }}.inventory{%- endraw %}" - vars: - hostname: __IP__ - hosts: - example-node: - count: 3 - host_groups: - - example - test-node: - count: 1 - host_groups: - - test - host_groups: - all: - vars: - ansible_user: root - ansible_private_key_file: "{% raw -%}{{ lookup('env', 'TESTLP') | default('/tmp', true) }}/CSS/keystore/css-central{%- endraw %}" - -dummy-topo: - topology: - topology_name: "dummy_cluster" # topology name - resource_groups: - - resource_group_name: "dummy" - resource_group_type: "dummy" - resource_definitions: - - name: "{{ distro | default('') }}web" - role: "dummy_node" - count: 3 - - name: "{{ distro | default('') }}test" - role: "dummy_node" - count: 1 - layout: - inventory_layout: - vars: - hostname: __IP__ - hosts: - example-node: - count: 3 - host_groups: - - example - test-node: - count: 1 - host_groups: - - test diff --git a/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/PinFile.json b/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/PinFile.json deleted file mode 100644 index a70d886..0000000 --- a/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/PinFile.json +++ /dev/null @@ -1 +0,0 @@ -{"dummy-prov2": {"topology": {"resource_groups": [{"resource_definitions": [{"count": 3, "name": "web", "role": "dummy_node"}], "resource_group_name": "dummy", "resource_group_type": "dummy"}], "topology_name": "dummy_cluster"}, "layout": {"inventory_layout": {"hosts": {"example-node": {"count": 3, "host_groups": ["example"]}}, "vars": {"hostname": "__IP__"}}}}} \ No newline at end of file diff --git a/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/README.rst b/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/README.rst deleted file mode 100644 index e98aafa..0000000 --- a/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/README.rst +++ /dev/null @@ -1,15 +0,0 @@ -Dummy -====== - -This workspace contains the basic tasks for provisioning and teardown of dummy clusters. - -As the name implies, dummy clusters do not provision any resources, but do run through -the provisioning steps for easy testing of linchpin features - -dummy-new -======== -Provisions a new set of dummy resources and creates a layout to go with them. The name of the inventory file is dynamically generated based on the uhash - -dummy-topo -======== -Another set of dummy resources with a slightly simpler layout diff --git a/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/inventories/dummy-new-44950f.inventory.inventory b/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/inventories/dummy-new-44950f.inventory.inventory deleted file mode 100644 index 2186d29..0000000 --- a/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/inventories/dummy-new-44950f.inventory.inventory +++ /dev/null @@ -1,18 +0,0 @@ -[all:vars] -ansible_user = root -ansible_private_key_file = {{ lookup('env', 'TESTLP') | default('/tmp', true) }}/CSS/keystore/css-central - -[test] -test hostname=test - -[example] -web-0 hostname=web-0 -web-1 hostname=web-1 -web-2 hostname=web-2 - -[all] -test hostname=test -web-2 hostname=web-2 -web-1 hostname=web-1 -web-0 hostname=web-0 - diff --git a/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/inventories/dummy_cluster.inventory b/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/inventories/dummy_cluster.inventory deleted file mode 100644 index c496581..0000000 --- a/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/inventories/dummy_cluster.inventory +++ /dev/null @@ -1,14 +0,0 @@ -[test] -test hostname=test - -[example] -web-0 hostname=web-0 -web-1 hostname=web-1 -web-2 hostname=web-2 - -[all] -test hostname=test -web-2 hostname=web-2 -web-1 hostname=web-1 -web-0 hostname=web-0 - diff --git a/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/resources/dummy_cluster.output b/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/resources/dummy_cluster.output deleted file mode 100644 index c2626a2..0000000 --- a/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/resources/dummy_cluster.output +++ /dev/null @@ -1 +0,0 @@ -[{"changed": true, "hosts": ["web-0", "web-1", "web-2"], "dummy_file": "/tmp/dummy.hosts", "failed": false, "resource_type": "dummy_res"}, {"changed": true, "hosts": ["test"], "dummy_file": "/tmp/dummy.hosts", "failed": false, "resource_type": "dummy_res"}] diff --git a/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/resources/linchpin.latest b/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/resources/linchpin.latest deleted file mode 100644 index a5f01d2..0000000 --- a/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/resources/linchpin.latest +++ /dev/null @@ -1 +0,0 @@ -{"1": {"action": "up", "targets": [{"dummy-new": {"1": {"rc": 0, "uhash": "44950f"}, "outputs": {"resources": [{"changed": true, "hosts": ["web-0", "web-1", "web-2"], "dummy_file": "/tmp/dummy.hosts", "failed": false, "resource_type": "dummy_res"}, {"changed": true, "hosts": ["test"], "dummy_file": "/tmp/dummy.hosts", "failed": false, "resource_type": "dummy_res"}], "inventory_path": ["/home/mankulka/development/linchpin/restylinchpin/restylinchpin/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/inventories/dummy-new-44950f.inventory.inventory"]}, "inputs": {"topology_data": {"topology_name": "dummy_cluster", "resource_groups": [{"resource_group_name": "dummy", "resource_group_type": "dummy", "resource_definitions": [{"name": "web", "role": "dummy_node", "count": 3}, {"name": "test", "role": "dummy_node", "count": 1}]}]}, "layout_data": {"inventory_layout": {"inventory_file": "{{ workspace }}/inventories/dummy-new-{{ uhash }}.inventory", "vars": {"hostname": "__IP__"}, "hosts": [{"name": "example-node", "count": 3, "host_groups": ["example"]}, {"name": "test-node", "count": 1, "host_groups": ["test"]}], "host_groups": {"all": {"vars": {"ansible_user": "root", "ansible_private_key_file": "{{ lookup('env', 'TESTLP') | default('/tmp', true) }}/CSS/keystore/css-central"}}}}}}}, "dummy-topo": {"1": {"rc": 0, "uhash": "86864e"}, "outputs": {"resources": [{"changed": true, "hosts": ["web-0", "web-1", "web-2"], "dummy_file": "/tmp/dummy.hosts", "failed": false, "resource_type": "dummy_res"}, {"changed": true, "hosts": ["test"], "dummy_file": "/tmp/dummy.hosts", "failed": false, "resource_type": "dummy_res"}], "inventory_path": ["/home/mankulka/development/linchpin/restylinchpin/restylinchpin/app/ /ecd70b0e-1348-43ca-9365-a634f00be88d_workspace1/dummy/inventories/dummy_cluster.inventory"]}, "inputs": {"topology_data": {"topology_name": "dummy_cluster", "resource_groups": [{"resource_group_name": "dummy", "resource_group_type": "dummy", "resource_definitions": [{"name": "web", "role": "dummy_node", "count": 3}, {"name": "test", "role": "dummy_node", "count": 1}]}]}, "layout_data": {"inventory_layout": {"vars": {"hostname": "__IP__"}, "hosts": [{"name": "example-node", "count": 3, "host_groups": ["example"]}, {"name": "test-node", "count": 1, "host_groups": ["test"]}]}}}}}]}} \ No newline at end of file diff --git a/app/ /tmp648b13a7-f048-45c5-8704-8bf8266e9d63_wk1/dummy/PinFile b/app/ /tmp648b13a7-f048-45c5-8704-8bf8266e9d63_wk1/dummy/PinFile deleted file mode 100644 index 585ff72..0000000 --- a/app/ /tmp648b13a7-f048-45c5-8704-8bf8266e9d63_wk1/dummy/PinFile +++ /dev/null @@ -1,60 +0,0 @@ ---- -dummy-new: - topology: - topology_name: "dummy_cluster" # topology name - resource_groups: - - resource_group_name: "dummy" - resource_group_type: "dummy" - resource_definitions: - - name: "{{ distro | default('') }}web" - role: "dummy_node" - count: 3 - - name: "{{ distro | default('') }}test" - role: "dummy_node" - count: 1 - layout: - inventory_layout: - inventory_file: "{% raw -%}{{ workspace }}/inventories/dummy-new-{{ uhash }}.inventory{%- endraw %}" - vars: - hostname: __IP__ - hosts: - example-node: - count: 3 - host_groups: - - example - test-node: - count: 1 - host_groups: - - test - host_groups: - all: - vars: - ansible_user: root - ansible_private_key_file: "{% raw -%}{{ lookup('env', 'TESTLP') | default('/tmp', true) }}/CSS/keystore/css-central{%- endraw %}" - -dummy-topo: - topology: - topology_name: "dummy_cluster" # topology name - resource_groups: - - resource_group_name: "dummy" - resource_group_type: "dummy" - resource_definitions: - - name: "{{ distro | default('') }}web" - role: "dummy_node" - count: 3 - - name: "{{ distro | default('') }}test" - role: "dummy_node" - count: 1 - layout: - inventory_layout: - vars: - hostname: __IP__ - hosts: - example-node: - count: 3 - host_groups: - - example - test-node: - count: 1 - host_groups: - - test diff --git a/app/ /tmp648b13a7-f048-45c5-8704-8bf8266e9d63_wk1/dummy/PinFile.json b/app/ /tmp648b13a7-f048-45c5-8704-8bf8266e9d63_wk1/dummy/PinFile.json deleted file mode 100644 index c3c942b..0000000 --- a/app/ /tmp648b13a7-f048-45c5-8704-8bf8266e9d63_wk1/dummy/PinFile.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "dummy": { - "topology": "dummy.json", - "layout": "dummy.json" - } -} - diff --git a/app/ /tmp648b13a7-f048-45c5-8704-8bf8266e9d63_wk1/dummy/README.rst b/app/ /tmp648b13a7-f048-45c5-8704-8bf8266e9d63_wk1/dummy/README.rst deleted file mode 100644 index e98aafa..0000000 --- a/app/ /tmp648b13a7-f048-45c5-8704-8bf8266e9d63_wk1/dummy/README.rst +++ /dev/null @@ -1,15 +0,0 @@ -Dummy -====== - -This workspace contains the basic tasks for provisioning and teardown of dummy clusters. - -As the name implies, dummy clusters do not provision any resources, but do run through -the provisioning steps for easy testing of linchpin features - -dummy-new -======== -Provisions a new set of dummy resources and creates a layout to go with them. The name of the inventory file is dynamically generated based on the uhash - -dummy-topo -======== -Another set of dummy resources with a slightly simpler layout diff --git a/app/__init__.py b/app/__init__.py index 7b74974..98a87d1 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -31,7 +31,7 @@ # loads defaults when config.yml does not exists or has been removed -WORKSPACE_DIR = config.get('workspace_path', '/') +WORKSPACE_DIR = config.get('workspace_path', '/tmp') LOGGER_FILE = config.get('logger_file_name', 'restylinchpin.log') DB_PATH = config.get('db_path', 'db.json') INVENTORY_PATH = config.get('inventory_path', '/dummy/inventories/*') @@ -42,7 +42,7 @@ ADMIN_USERNAME = config.get('admin_username', 'admin') ADMIN_PASSWORD = config.get('admin_password', 'password') ADMIN_EMAIL = config.get('admin_email', 'email') -CREDS_PATH = config.get('creds_path', '/') +CREDS_PATH = config.get('creds_path', '/tmp') # URL for exposing Swagger UI (without trailing '/') SWAGGER_URL = '/api/docs' @@ -722,7 +722,8 @@ def get_linchpin_inventory(current_user, identity) -> Response: return jsonify(message=response.NOT_FOUND) if not current_user['admin']: workspace = db_con.db_search_identity(identity) - if not db_con.db_search(workspace['name'], current_user['admin'], + if not db_con.db_search(workspace['name'], + current_user['admin'], current_user['username']): return jsonify(message=response.NOT_FOUND) data = request.json @@ -748,26 +749,38 @@ def get_linchpin_inventory(current_user, identity) -> Response: return jsonify(status=errors.ERROR_STATUS, message=str(e)) -@app.route('/api/v1.0/credentials', methods=['POST']) +@app.route('/api/v1.0/users//credentials', methods=['POST']) @auth_required -def upload_credentials(current_user) -> Response: +def upload_credentials(current_user, username) -> Response: + """ + POST request for uploading credentials to user account + RequestBody: { file : text or attached file + file_name : name, + encrypted: boolean, + vault_pass: ansible vault password + } + return : response with successful credential upload status + """ db_con = get_connection_users(DB_PATH) try: - user = db_con.db_search_name(current_user['username']) - if not user: - return jsonify(message=response.USER_NOT_FOUND) + user = db_con.db_search_name(username) + if not current_user['username'] == username \ + and not current_user['admin']: + return jsonify(message=errors.UNAUTHORIZED_REQUEST) file_name = request.form['file_name'] encrypted = request.form['encrypted'] - if request.form["creds_folder_name"]: + if 'creds_folder_name' in request.form: creds_folder = request.form["creds_folder_name"] else: - if current_user['creds_folder'] is None: - creds_folder = current_user['username'] + "_" + str(uuid.uuid4()) - db_con.db_update_creds_folder(current_user['username'], + if user['creds_folder'] is None: + print("now here") + creds_folder = username + "_" + str(uuid.uuid4()) + db_con.db_update_creds_folder(username, creds_folder) os.makedirs(WORKSPACE_PATH + CREDS_PATH + creds_folder) else: - creds_folder = current_user['creds_folder'] + print("expecting here") + creds_folder = user['creds_folder'] if request.files: file = request.files["file"] file_read = file.read() @@ -775,15 +788,19 @@ def upload_credentials(current_user) -> Response: file_read = request.form["file"] if encrypted.lower() in ("true", "t"): if request.files: - with open(WORKSPACE_PATH + CREDS_PATH + creds_folder + "/" + file_name + ".yml", 'wb') as yaml_file: + with open(WORKSPACE_PATH + CREDS_PATH + creds_folder + + "/" + file_name + ".yml", 'wb') as yaml_file: yaml_file.write(file_read) else: - with open(WORKSPACE_PATH + CREDS_PATH + creds_folder + "/" + file_name + ".yml", 'w') as yaml_file: + with open(WORKSPACE_PATH + CREDS_PATH + creds_folder + + "/" + file_name + ".yml", 'w') as yaml_file: yaml_file.write(file_read) else: vault_pass = request.form['vault_pass'] vault = Vault(vault_pass) - vault.dump(file_read, open(WORKSPACE_PATH + CREDS_PATH + creds_folder + "/" + file_name + ".yml", 'wb')) + vault.dump(file_read, open(WORKSPACE_PATH + CREDS_PATH + + creds_folder + "/" + file_name + + ".yml", 'wb')) return jsonify(message=response.CREDENTIALS_UPLOADED) except (KeyError, ValueError, TypeError): return jsonify(status=errors.ERROR_STATUS, @@ -793,18 +810,24 @@ def upload_credentials(current_user) -> Response: return jsonify(status=errors.ERROR_STATUS, message=str(e)) -@app.route('/api/v1.0/credentials/', methods=['GET']) +@app.route('/api/v1.0/users//credentials/', + methods=['GET']) @auth_required -def get_credentials(current_user, file_name) -> Response: +def get_credentials(current_user, file_name, username) -> Response: + """ + GET request to retrieve credentials from a credential file + return : response with encrypted credentials from file + """ db_con = get_connection_users(DB_PATH) try: - user = db_con.db_search_name(current_user['username']) - if not user: - return jsonify(message=response.USER_NOT_FOUND) - if not os.listdir(WORKSPACE_PATH + CREDS_PATH + current_user['creds_folder']). \ + user = db_con.db_search_name(username) + if not current_user['username'] == username \ + and not current_user['admin']: + return jsonify(message=errors.UNAUTHORIZED_REQUEST) + if not os.listdir(WORKSPACE_PATH + CREDS_PATH + user['creds_folder']). \ __contains__(file_name): return jsonify(message=response.CREDENTIALS_FILE_NOT_FOUND) - with open(WORKSPACE_PATH + CREDS_PATH + current_user['creds_folder'] + + with open(WORKSPACE_PATH + CREDS_PATH + user['creds_folder'] + "/" + file_name, 'r') as data: credentials = data.read().replace('\n', ' ') return jsonify(credentials=credentials) @@ -816,16 +839,26 @@ def get_credentials(current_user, file_name) -> Response: return jsonify(status=errors.ERROR_STATUS, message=str(e)) -@app.route('/api/v1.0/credentials/', methods=['PUT']) +@app.route('/api/v1.0/users//credentials/', + methods=['PUT']) @auth_required -def update_credentials(current_user, file_name) -> Response: +def update_credentials(current_user, username, file_name) -> Response: + """ + PUT request for updating credentials to user account + RequestBody: { file : updated text or updated attached file + encrypted: boolean, + vault_pass: ansible vault password + } + return : response with successful credential update status + """ db_con = get_connection_users(DB_PATH) try: - user = db_con.db_search_name(current_user['username']) - if not user: - return jsonify(message=response.USER_NOT_FOUND) + user = db_con.db_search_name(username) + if not current_user['username'] == username \ + and not current_user['admin']: + return jsonify(message=errors.UNAUTHORIZED_REQUEST) if 'creds_folder_name' not in request.form: - creds_folder = current_user['creds_folder'] + creds_folder = user['creds_folder'] else: creds_folder = request.form['creds_folder_name'] if not os.listdir(WORKSPACE_PATH + CREDS_PATH + creds_folder). \ @@ -839,39 +872,47 @@ def update_credentials(current_user, file_name) -> Response: encrypted = request.form['encrypted'] if encrypted.lower() in ("true", "t"): if request.files: - with open(WORKSPACE_PATH + CREDS_PATH + creds_folder + "/" + file_name, 'wb') as yaml_file: + with open(WORKSPACE_PATH + CREDS_PATH + creds_folder + + "/" + file_name, 'wb') as yaml_file: yaml_file.write(file_read) else: - with open(WORKSPACE_PATH + CREDS_PATH + creds_folder + "/" + file_name, 'w') as yaml_file: + with open(WORKSPACE_PATH + CREDS_PATH + creds_folder + + "/" + file_name, 'w') as yaml_file: yaml_file.write(file_read) else: vault_pass = request.form['vault_pass'] vault = Vault(vault_pass) - vault.dump(file_read, open(WORKSPACE_PATH + CREDS_PATH + creds_folder + "/" + file_name, 'wb')) + vault.dump(file_read, open(WORKSPACE_PATH + CREDS_PATH + + creds_folder + "/" + file_name, 'wb')) return jsonify(message=response.CREDENTIALS_UPDATED) except Exception as e: app.logger.error(e) return jsonify(status=errors.ERROR_STATUS, message=str(e)) -@app.route('/api/v1.0/credentials/', methods=['DELETE']) +@app.route('/api/v1.0/users//credentials/', + methods=['DELETE']) @auth_required -def delete_credentials(current_user, file_name) -> Response: +def delete_credentials(current_user, username, file_name) -> Response: + """ + DELETE request to delete a credential file + return : response with successful delete status + """ db_con = get_connection_users(DB_PATH) try: - user = db_con.db_search_name(current_user['username']) - if not user: - return jsonify(message=response.USER_NOT_FOUND) + user = db_con.db_search_name(username) + if not current_user['username'] == username \ + and not current_user['admin']: + return jsonify(message=errors.UNAUTHORIZED_REQUEST) if 'creds_folder_name' not in request.form: - creds_folder = current_user['creds_folder'] + creds_folder = user['creds_folder'] else: creds_folder = request.form['creds_folder_name'] - for w in os.listdir(WORKSPACE_PATH + CREDS_PATH + creds_folder): - print(w) + for w in os.listdir(WORKSPACE_PATH + CREDS_PATH + + creds_folder): if w == file_name: - print("here") - os.remove(WORKSPACE_PATH + CREDS_PATH + creds_folder + "/" + w) - print("deleted") + os.remove(WORKSPACE_PATH + CREDS_PATH + + creds_folder + "/" + w) return jsonify(status=response.CREDENTIALS_DELETED, mimetype='application/json') return jsonify(status=response.CREDENTIALS_FILE_NOT_FOUND) diff --git a/app/config.yml b/app/config.yml index 106721d..2a236dc 100644 --- a/app/config.yml +++ b/app/config.yml @@ -1,6 +1,6 @@ # place where all the created workspaces are stored # workspace_path --> place where are the workspaces are created and stored -workspace_path: / +workspace_path: /tmp # name or path of logger file logger_file_name: restylinchpin.log # path to workspace tinydb source file @@ -18,4 +18,4 @@ admin_username: admin admin_password: password admin_email: admin@xyz # path to creds folder -creds_path: / \ No newline at end of file +creds_path: /tmp \ No newline at end of file diff --git a/app/data_access_layer/UserRestDB.py b/app/data_access_layer/UserRestDB.py index a52df56..64a9821 100644 --- a/app/data_access_layer/UserRestDB.py +++ b/app/data_access_layer/UserRestDB.py @@ -38,7 +38,7 @@ def db_search_name(self, username) -> List[Dict]: param username """ user = Query() - return self.table.search(user.username == username) + return self.table.search(user.username == username)[0] def db_list_all(self) -> List[Dict]: """ @@ -116,6 +116,7 @@ def db_update(self, username, updated_username, password_hash, 'password': password_hash, 'email': email}, user.username == username) - def db_update_creds_folder(self, username, creds_folder): + def db_update_creds_folder(self, username, creds_folder): user = Query() - self.table.update({'creds_folder': creds_folder}, user.username == username) + self.table.update({'creds_folder': creds_folder}, + user.username == username) diff --git a/requirements.txt b/requirements.txt index 6595851..74c1e19 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ Flask>=1.0.3 linchpin<=1.7.5 flask-swagger-ui>=3.20.9 flake8>=3.7.7 + ansible-vault>=1.2.0 \ No newline at end of file From d6fcf8b60eac26836d42559903b1c4acccd319ed Mon Sep 17 00:00:00 2001 From: Mansi Date: Mon, 5 Aug 2019 10:12:42 -0400 Subject: [PATCH 05/20] code cleanup remove print statements, added param write --- app/__init__.py | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 98a87d1..6e7ba4e 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -42,7 +42,7 @@ ADMIN_USERNAME = config.get('admin_username', 'admin') ADMIN_PASSWORD = config.get('admin_password', 'password') ADMIN_EMAIL = config.get('admin_email', 'email') -CREDS_PATH = config.get('creds_path', '/tmp') +CREDS_PATH = config.get('creds_path', '/') # URL for exposing Swagger UI (without trailing '/') SWAGGER_URL = '/api/docs' @@ -773,29 +773,25 @@ def upload_credentials(current_user, username) -> Response: creds_folder = request.form["creds_folder_name"] else: if user['creds_folder'] is None: - print("now here") creds_folder = username + "_" + str(uuid.uuid4()) db_con.db_update_creds_folder(username, creds_folder) os.makedirs(WORKSPACE_PATH + CREDS_PATH + creds_folder) else: - print("expecting here") creds_folder = user['creds_folder'] if request.files: file = request.files["file"] file_read = file.read() + write = 'wb' else: file_read = request.form["file"] + write = 'w' if encrypted.lower() in ("true", "t"): - if request.files: - with open(WORKSPACE_PATH + CREDS_PATH + creds_folder + - "/" + file_name + ".yml", 'wb') as yaml_file: - yaml_file.write(file_read) - else: - with open(WORKSPACE_PATH + CREDS_PATH + creds_folder + - "/" + file_name + ".yml", 'w') as yaml_file: - yaml_file.write(file_read) + with open(WORKSPACE_PATH + CREDS_PATH + creds_folder + + "/" + file_name + ".yml", write) as yaml_file: + yaml_file.write(file_read) else: + print("here") vault_pass = request.form['vault_pass'] vault = Vault(vault_pass) vault.dump(file_read, open(WORKSPACE_PATH + CREDS_PATH + @@ -867,18 +863,15 @@ def update_credentials(current_user, username, file_name) -> Response: if request.files: file = request.files["file"] file_read = file.read() + write = 'wb' else: file_read = request.form["file"] + write = 'w' encrypted = request.form['encrypted'] if encrypted.lower() in ("true", "t"): - if request.files: - with open(WORKSPACE_PATH + CREDS_PATH + creds_folder + - "/" + file_name, 'wb') as yaml_file: - yaml_file.write(file_read) - else: - with open(WORKSPACE_PATH + CREDS_PATH + creds_folder + - "/" + file_name, 'w') as yaml_file: - yaml_file.write(file_read) + with open(WORKSPACE_PATH + CREDS_PATH + creds_folder + + "/" + file_name, write) as yaml_file: + yaml_file.write(file_read) else: vault_pass = request.form['vault_pass'] vault = Vault(vault_pass) From 90cf737ec904fc65f00ad546b8fae8e78f98ec76 Mon Sep 17 00:00:00 2001 From: Mansi Date: Tue, 6 Aug 2019 14:40:26 -0400 Subject: [PATCH 06/20] Added creds_path in linchpin up,destroy --- app/__init__.py | 31 +++++++++++++++++++++++-------- app/utils/__init__.py | 17 ++++++++++++++--- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 6e7ba4e..e0a4274 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -42,7 +42,7 @@ ADMIN_USERNAME = config.get('admin_username', 'admin') ADMIN_PASSWORD = config.get('admin_password', 'password') ADMIN_EMAIL = config.get('admin_email', 'email') -CREDS_PATH = config.get('creds_path', '/') +CREDS_PATH = config.get('creds_path', '/tmp') # URL for exposing Swagger UI (without trailing '/') SWAGGER_URL = '/api/docs' @@ -502,9 +502,9 @@ def linchpin_fetch_workspace(current_user) -> Response: message=errors.KEY_ERROR_PARAMS_FETCH) -@app.route('/api/v1.0/workspaces/up', methods=['POST']) +@app.route('/api/v1.0/users//workspaces/up', methods=['POST']) @auth_required -def linchpin_up(current_user) -> Response: +def linchpin_up(current_user, username) -> Response: """ POST request route for provisioning workspaces/pinFile already created @@ -518,8 +518,13 @@ def linchpin_up(current_user) -> Response: """ identity = None db_con = get_connection(DB_PATH) + db_con_users = get_connection_users(DB_PATH) try: workspace = db_con.db_search_username(current_user['username']) + user = db_con_users.db_search_name(username) + if not current_user['username'] == username \ + and not current_user['admin']: + return jsonify(message=errors.UNAUTHORIZED_REQUEST) if not current_user['admin'] and not workspace: return jsonify(message=response.NOT_FOUND) data = request.json # Get request body @@ -528,6 +533,7 @@ def linchpin_up(current_user) -> Response: identity = data['id'] if not current_user['admin']: workspace = db_con.db_search_identity(identity) + if not db_con.db_search(workspace['name'], current_user['admin'], current_user['username']): @@ -535,7 +541,8 @@ def linchpin_up(current_user) -> Response: if not os.path.exists(WORKSPACE_PATH + "/" + identity): return jsonify(status=response.NOT_FOUND) cmd = create_cmd_workspace(data, identity, "up", - WORKSPACE_PATH, WORKSPACE_DIR) + WORKSPACE_PATH, WORKSPACE_DIR, + user['creds_folder']) elif provision_type == "pinfile": if 'name' in data: identity = str(uuid.uuid4()) + "_" + data['name'] @@ -549,7 +556,8 @@ def linchpin_up(current_user) -> Response: response.WORKSPACE_REQUESTED, current_user['username']) cmd = create_cmd_up_pinfile(data, identity, WORKSPACE_PATH, - WORKSPACE_DIR, PINFILE_JSON_PATH) + WORKSPACE_DIR, PINFILE_JSON_PATH, + user['creds_folder']) else: raise ValueError output = subprocess.Popen(cmd, stdout=subprocess.PIPE) @@ -578,9 +586,9 @@ def linchpin_up(current_user) -> Response: return jsonify(status=errors.ERROR_STATUS, message=str(e)) -@app.route('/api/v1.0/workspaces/destroy', methods=['POST']) +@app.route('/api/v1.0/users//workspaces/destroy', methods=['POST']) @auth_required -def linchpin_destroy(current_user) -> Response: +def linchpin_destroy(current_user, username) -> Response: """ POST request route for destroying workspaces/resources already created or provisioned @@ -589,8 +597,14 @@ def linchpin_destroy(current_user) -> Response: """ identity = None db_con = get_connection(DB_PATH) + db_con_users = get_connection_users(DB_PATH) try: workspace = db_con.db_search_username(current_user['username']) + user = db_con_users.db_search_name(username) + if not current_user['username'] == username \ + and not current_user['admin']: + return jsonify(message=errors.UNAUTHORIZED_REQUEST) + print(user) if not current_user['admin'] and not workspace: return jsonify(message=response.NOT_FOUND) data = request.json # Get request body @@ -601,7 +615,8 @@ def linchpin_destroy(current_user) -> Response: current_user['username']): return jsonify(message=response.NOT_FOUND) cmd = create_cmd_workspace(data, identity, "destroy", WORKSPACE_PATH, - WORKSPACE_DIR) + WORKSPACE_DIR, user['creds_folder']) + print(cmd) output = subprocess.Popen(cmd, stdout=subprocess.PIPE) output.communicate() db_con.db_update(identity, response.DESTROY_STATUS_SUCCESS) diff --git a/app/utils/__init__.py b/app/utils/__init__.py index 01ad95b..ea23df9 100644 --- a/app/utils/__init__.py +++ b/app/utils/__init__.py @@ -74,7 +74,8 @@ def create_fetch_cmd(data, identity, workspace_dir) -> List[str]: def create_cmd_workspace(data, identity, action, - workspace_path, workspace_dir) -> List[str]: + workspace_path, workspace_dir, + creds_folder) -> List[str]: """ Creates a list to feed the subprocess for provisioning/ destroying existing workspaces @@ -89,6 +90,10 @@ def create_cmd_workspace(data, identity, action, else: check_path = identity cmd = ["linchpin", "-w " + workspace_dir + check_path] + if 'creds_path' in data: + cmd.extend(("--creds-path", data['creds_path'])) + else: + cmd.extend(("--creds-path", creds_folder)) if 'pinfile_name' in data: cmd.extend(("-p", data['pinfile_name'])) pinfile_name = data['pinfile_name'] @@ -111,7 +116,8 @@ def create_cmd_up_pinfile(data, identity, workspace_path, workspace_dir, - pinfile_json_path) -> List[str]: + pinfile_json_path, + creds_folder) -> List[str]: """ Creates a list to feed the subprocess for provisioning new workspaces instantiated using a pinfile @@ -124,7 +130,12 @@ def create_cmd_up_pinfile(data, with open(json_pinfile_path, 'w') as json_data: json.dump(pinfile_content, json_data) cmd = ["linchpin", "-w " + workspace_dir + identity + "/dummy", "-p" + - "PinFile.json", "up"] + "PinFile.json"] + if 'creds_path' in data: + cmd.extend(("--creds-path", data['creds_path'])) + else: + cmd.extend(("--creds-path", creds_folder)) + cmd.append("up") if 'inventory_format' in data: cmd.extend(("--if", data['inventory_format'])) return cmd From fe3fe4d39e7d8bbe9dc07ae03ae5cf3030163888 Mon Sep 17 00:00:00 2001 From: Mansi Date: Tue, 6 Aug 2019 15:30:59 -0400 Subject: [PATCH 07/20] Update readme, remove print statements --- README.md | 28 ++++++++++++++++++++++++++-- app/__init__.py | 3 --- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b78f350..5fb0448 100644 --- a/README.md +++ b/README.md @@ -103,9 +103,33 @@ username=username,
password= hashed_password,
email=email,
status=OK
+}

+Upload/create credentials
+POST /users/username/credentials
+RequestBody:
+{ file : text or attached file
+file_name : name,
+encrypted: boolean,
+vault_pass: ansible vault password
}
- - +return : response with successful credential upload status
+
+List credentials
+GET /users/username/credentials/file_name
+return : response with encrypted credentials from file
+ +Delete credentials
+DELETE /users/username/credentials/file-name
+return : response with successful delete status
+
+Update credentials
+PUT /users/username/credentials/file_name
+RequestBody: { file : updated text or updated attached file
+encrypted: boolean,
+vault_pass: ansible vault password
+}
+return : response with successful credential update status
+
## Linchpin Project LinchPin is a simple cloud orchestration tool. Its intended purpose is managing cloud resources across multiple infrastructures. These resources can be provisioned, decommissioned, and configured all using declarative data and a simple command-line interface. diff --git a/app/__init__.py b/app/__init__.py index e0a4274..f65f14d 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -604,7 +604,6 @@ def linchpin_destroy(current_user, username) -> Response: if not current_user['username'] == username \ and not current_user['admin']: return jsonify(message=errors.UNAUTHORIZED_REQUEST) - print(user) if not current_user['admin'] and not workspace: return jsonify(message=response.NOT_FOUND) data = request.json # Get request body @@ -616,7 +615,6 @@ def linchpin_destroy(current_user, username) -> Response: return jsonify(message=response.NOT_FOUND) cmd = create_cmd_workspace(data, identity, "destroy", WORKSPACE_PATH, WORKSPACE_DIR, user['creds_folder']) - print(cmd) output = subprocess.Popen(cmd, stdout=subprocess.PIPE) output.communicate() db_con.db_update(identity, response.DESTROY_STATUS_SUCCESS) @@ -806,7 +804,6 @@ def upload_credentials(current_user, username) -> Response: "/" + file_name + ".yml", write) as yaml_file: yaml_file.write(file_read) else: - print("here") vault_pass = request.form['vault_pass'] vault = Vault(vault_pass) vault.dump(file_read, open(WORKSPACE_PATH + CREDS_PATH + From 44940e414c28265bf9e602bf0f1fcae0fb41c055 Mon Sep 17 00:00:00 2001 From: Mansi Date: Tue, 6 Aug 2019 17:07:05 -0400 Subject: [PATCH 08/20] Updated path for creds path --- app/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index f65f14d..80a43d3 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -529,6 +529,7 @@ def linchpin_up(current_user, username) -> Response: return jsonify(message=response.NOT_FOUND) data = request.json # Get request body provision_type = data['provision_type'] + creds_path = WORKSPACE_PATH + CREDS_PATH + user['creds_folder'] if provision_type == "workspace": identity = data['id'] if not current_user['admin']: @@ -542,7 +543,7 @@ def linchpin_up(current_user, username) -> Response: return jsonify(status=response.NOT_FOUND) cmd = create_cmd_workspace(data, identity, "up", WORKSPACE_PATH, WORKSPACE_DIR, - user['creds_folder']) + creds_path) elif provision_type == "pinfile": if 'name' in data: identity = str(uuid.uuid4()) + "_" + data['name'] @@ -557,7 +558,7 @@ def linchpin_up(current_user, username) -> Response: current_user['username']) cmd = create_cmd_up_pinfile(data, identity, WORKSPACE_PATH, WORKSPACE_DIR, PINFILE_JSON_PATH, - user['creds_folder']) + creds_path) else: raise ValueError output = subprocess.Popen(cmd, stdout=subprocess.PIPE) @@ -608,13 +609,14 @@ def linchpin_destroy(current_user, username) -> Response: return jsonify(message=response.NOT_FOUND) data = request.json # Get request body identity = data['id'] + creds_path = WORKSPACE_PATH + CREDS_PATH + user['creds_folder'] if not current_user['admin']: workspace = db_con.db_search_identity(identity) if not db_con.db_search(workspace['name'], current_user['admin'], current_user['username']): return jsonify(message=response.NOT_FOUND) cmd = create_cmd_workspace(data, identity, "destroy", WORKSPACE_PATH, - WORKSPACE_DIR, user['creds_folder']) + WORKSPACE_DIR, creds_path) output = subprocess.Popen(cmd, stdout=subprocess.PIPE) output.communicate() db_con.db_update(identity, response.DESTROY_STATUS_SUCCESS) From 061ababe36303953b2b75672dadfbc12d8c698d3 Mon Sep 17 00:00:00 2001 From: Mansi Date: Wed, 7 Aug 2019 09:54:34 -0400 Subject: [PATCH 09/20] fixing travis ci errors --- app/utils/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/app/utils/__init__.py b/app/utils/__init__.py index ea23df9..f27ebb6 100644 --- a/app/utils/__init__.py +++ b/app/utils/__init__.py @@ -75,13 +75,14 @@ def create_fetch_cmd(data, identity, workspace_dir) -> List[str]: def create_cmd_workspace(data, identity, action, workspace_path, workspace_dir, - creds_folder) -> List[str]: + creds_folder_path) -> List[str]: """ Creates a list to feed the subprocess for provisioning/ destroying existing workspaces :param data: JSON data from POST requestBody :param identity: unique uuid_name assigned to the workspace :param action: up or destroy action + :param creds_folder_path: path to the credentials folder :return a list for the subprocess to run """ if 'pinfile_path' in data: @@ -93,7 +94,7 @@ def create_cmd_workspace(data, identity, action, if 'creds_path' in data: cmd.extend(("--creds-path", data['creds_path'])) else: - cmd.extend(("--creds-path", creds_folder)) + cmd.extend(("--creds-path", creds_folder_path)) if 'pinfile_name' in data: cmd.extend(("-p", data['pinfile_name'])) pinfile_name = data['pinfile_name'] @@ -117,12 +118,13 @@ def create_cmd_up_pinfile(data, workspace_path, workspace_dir, pinfile_json_path, - creds_folder) -> List[str]: + creds_folder_path) -> List[str]: """ Creates a list to feed the subprocess for provisioning new workspaces instantiated using a pinfile :param data: JSON data from POST requestBody :param identity: unique uuid_name assigned to the workspace + :param creds_folder_path: path to the credentials folder :return a list for the subprocess to run """ pinfile_content = data['pinfile_content'] @@ -134,7 +136,7 @@ def create_cmd_up_pinfile(data, if 'creds_path' in data: cmd.extend(("--creds-path", data['creds_path'])) else: - cmd.extend(("--creds-path", creds_folder)) + cmd.extend(("--creds-path", creds_folder_path)) cmd.append("up") if 'inventory_format' in data: cmd.extend(("--if", data['inventory_format'])) From ebb9c1ae1d1fa1fd5468f79e7726782662c5cd8b Mon Sep 17 00:00:00 2001 From: Mansi Date: Wed, 21 Aug 2019 09:31:01 -0400 Subject: [PATCH 10/20] Added unit tests --- app/tests/UnitTest.py | 55 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 app/tests/UnitTest.py diff --git a/app/tests/UnitTest.py b/app/tests/UnitTest.py new file mode 100644 index 0000000..d4bdefb --- /dev/null +++ b/app/tests/UnitTest.py @@ -0,0 +1,55 @@ +import unittest +import app +from app.data_access_layer import RestDB +from app.data_access_layer import UserRestDB + +BASE_URL = 'http://localhost:5000/api/v1.0' + + +class UnitTest(unittest.TestCase): + + def setUp(self): + self.app = app.app.test_client() + self.app.testing = True + + def test_db_create_admin_user(self): + app.create_admin_user(app.DB_PATH, app.ADMIN_USERNAME, + app.ADMIN_PASSWORD, app.ADMIN_EMAIL) + self.assertEqual("admin", app.get_connection_users(app.DB_PATH).db_search_name("admin")['username']) + + def test_get_connection(self): + self.assertIsInstance(app.get_connection(app.DB_PATH), RestDB.RestDB) + + def test_get_connection_users(self): + self.assertIsInstance(app.get_connection_users(app.DB_PATH), UserRestDB.UserRestDB) + + def test_create_fetch_cmd(self): + data = {"name": "test", "url": "www.github.com/CentOS-PaaS-SIG/linchpin", + "rootfolder":"/"} + identity = "test123" + self.assertEqual(app.create_fetch_cmd(data, identity, app.WORKSPACE_DIR), + ["linchpin", "-w " + app.WORKSPACE_DIR + identity, "fetch", "--root", data['rootfolder'], + data['url']]) + + def test_create_cmd_workspace(self): + data = {"id": "test123", + "provision_type": "workspace", + "pinfile_path": "/dummy/" + } + action = "up" + self.assertEqual(app.create_cmd_workspace(data, data['id'], action, + app.WORKSPACE_PATH, app.WORKSPACE_DIR, + app.CREDS_PATH), ["linchpin", "-w " + app.WORKSPACE_DIR + data['id'] + data['pinfile_path'], + "--creds-path", "/", "up"]) + + def test_create_cmd_up_pinfile(self): + data = {"id": "UnitTest", + "provision_type": "pinfile", + "pinfile_content":{"test":"pinfile_data"}} + self.assertEqual(app.create_cmd_up_pinfile(data, data['id'], app.WORKSPACE_DIR, app.CREDS_PATH), + ["linchpin", "-w " + app.WORKSPACE_DIR + data['id'] + "/dummy", "-p" + + "PinFile.json", "--creds-path", "/", "up"]) + + +if __name__ == '__main__': + unittest.main() From 598243c9cce31184ce3e729843e38987adad67ac Mon Sep 17 00:00:00 2001 From: Mansi Date: Wed, 21 Aug 2019 09:37:00 -0400 Subject: [PATCH 11/20] code refactoring changes --- app/__init__.py | 27 ++++++++++++++++++++++----- app/utils/__init__.py | 14 +------------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 80a43d3..f4ceb33 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -31,7 +31,7 @@ # loads defaults when config.yml does not exists or has been removed -WORKSPACE_DIR = config.get('workspace_path', '/tmp') +WORKSPACE_DIR = config.get('workspace_path', '/tests/') LOGGER_FILE = config.get('logger_file_name', 'restylinchpin.log') DB_PATH = config.get('db_path', 'db.json') INVENTORY_PATH = config.get('inventory_path', '/dummy/inventories/*') @@ -42,7 +42,7 @@ ADMIN_USERNAME = config.get('admin_username', 'admin') ADMIN_PASSWORD = config.get('admin_password', 'password') ADMIN_EMAIL = config.get('admin_email', 'email') -CREDS_PATH = config.get('creds_path', '/tmp') +CREDS_PATH = config.get('creds_path', '/') # URL for exposing Swagger UI (without trailing '/') SWAGGER_URL = '/api/docs' @@ -483,7 +483,7 @@ def linchpin_fetch_workspace(current_user) -> Response: else: output = subprocess.Popen(cmd, stdout=subprocess.PIPE) output.communicate() - if check_workspace_empty(identity, WORKSPACE_PATH): + if check_workspace_empty(identity, WORKSPACE_PATH, WORKSPACE_DIR): db_con.db_update(identity, response.WORKSPACE_FAILED) return jsonify(status=response.EMPTY_WORKSPACE) @@ -541,6 +541,19 @@ def linchpin_up(current_user, username) -> Response: return jsonify(message=response.NOT_FOUND) if not os.path.exists(WORKSPACE_PATH + "/" + identity): return jsonify(status=response.NOT_FOUND) + if 'pinfile_path' in data: + pinfile_path = data['pinfile_path'] + check_path = identity + pinfile_path + else: + check_path = identity + if 'pinfile_name' in data: + pinfile_name = data['pinfile_name'] + else: + pinfile_name = "PinFile" + if not check_workspace_has_pinfile(check_path, pinfile_name, + WORKSPACE_PATH): + return jsonify(status=response.PINFILE_NOT_FOUND) + cmd = create_cmd_workspace(data, identity, "up", WORKSPACE_PATH, WORKSPACE_DIR, creds_path) @@ -549,6 +562,8 @@ def linchpin_up(current_user, username) -> Response: identity = str(uuid.uuid4()) + "_" + data['name'] else: identity = str(uuid.uuid4()) + pinfile_content = data['pinfile_content'] + json_pinfile_path = WORKSPACE_PATH + "/" + identity + PINFILE_JSON_PATH precmd = ["linchpin", "-w " + WORKSPACE_DIR + identity + "/", "init"] output = subprocess.Popen(precmd, stdout=subprocess.PIPE) @@ -556,8 +571,10 @@ def linchpin_up(current_user, username) -> Response: db_con.db_insert_no_name(identity, response.WORKSPACE_REQUESTED, current_user['username']) - cmd = create_cmd_up_pinfile(data, identity, WORKSPACE_PATH, - WORKSPACE_DIR, PINFILE_JSON_PATH, + with open(json_pinfile_path, 'w') as json_data: + json.dump(pinfile_content, json_data) + cmd = create_cmd_up_pinfile(data, identity, + WORKSPACE_DIR, creds_path) else: raise ValueError diff --git a/app/utils/__init__.py b/app/utils/__init__.py index f27ebb6..192176c 100644 --- a/app/utils/__init__.py +++ b/app/utils/__init__.py @@ -97,12 +97,6 @@ def create_cmd_workspace(data, identity, action, cmd.extend(("--creds-path", creds_folder_path)) if 'pinfile_name' in data: cmd.extend(("-p", data['pinfile_name'])) - pinfile_name = data['pinfile_name'] - else: - pinfile_name = "PinFile" - if not check_workspace_has_pinfile(check_path, pinfile_name, - workspace_path): - return jsonify(status=response.PINFILE_NOT_FOUND) cmd.append(action) if 'tx_id' in data: cmd.extend(("-t", data['tx_id'])) @@ -115,9 +109,7 @@ def create_cmd_workspace(data, identity, action, def create_cmd_up_pinfile(data, identity, - workspace_path, workspace_dir, - pinfile_json_path, creds_folder_path) -> List[str]: """ Creates a list to feed the subprocess for provisioning @@ -127,10 +119,6 @@ def create_cmd_up_pinfile(data, :param creds_folder_path: path to the credentials folder :return a list for the subprocess to run """ - pinfile_content = data['pinfile_content'] - json_pinfile_path = workspace_path + "/" + identity + pinfile_json_path - with open(json_pinfile_path, 'w') as json_data: - json.dump(pinfile_content, json_data) cmd = ["linchpin", "-w " + workspace_dir + identity + "/dummy", "-p" + "PinFile.json"] if 'creds_path' in data: @@ -153,7 +141,7 @@ def check_workspace_has_pinfile(name, pinfile_name, workspace_path) -> bool: return os.listdir(workspace_path + "/" + name).__contains__(pinfile_name) -def check_workspace_empty(name, workspace_path) -> bool: +def check_workspace_empty(name, workspace_path, working_dir) -> bool: """ Verifies if a workspace fetched/created is empty :param name: name of the workspace to be verified From 30510c74bac43b203c3a1bd12aca7065b0c5d5d9 Mon Sep 17 00:00:00 2001 From: Mansi Date: Wed, 21 Aug 2019 10:22:08 -0400 Subject: [PATCH 12/20] corrected config paths --- app/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index f4ceb33..5a4f898 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -31,7 +31,7 @@ # loads defaults when config.yml does not exists or has been removed -WORKSPACE_DIR = config.get('workspace_path', '/tests/') +WORKSPACE_DIR = config.get('workspace_path', '/tmp') LOGGER_FILE = config.get('logger_file_name', 'restylinchpin.log') DB_PATH = config.get('db_path', 'db.json') INVENTORY_PATH = config.get('inventory_path', '/dummy/inventories/*') @@ -42,7 +42,7 @@ ADMIN_USERNAME = config.get('admin_username', 'admin') ADMIN_PASSWORD = config.get('admin_password', 'password') ADMIN_EMAIL = config.get('admin_email', 'email') -CREDS_PATH = config.get('creds_path', '/') +CREDS_PATH = config.get('creds_path', '/tmp') # URL for exposing Swagger UI (without trailing '/') SWAGGER_URL = '/api/docs' From a59e0f7b5240da4cbd0516d09581ef47d2b5b19f Mon Sep 17 00:00:00 2001 From: Mansi Date: Wed, 21 Aug 2019 10:30:13 -0400 Subject: [PATCH 13/20] fixed flake8 errors --- app/__init__.py | 9 ++++++--- app/utils/__init__.py | 3 --- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 5a4f898..ecd3e86 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -483,7 +483,8 @@ def linchpin_fetch_workspace(current_user) -> Response: else: output = subprocess.Popen(cmd, stdout=subprocess.PIPE) output.communicate() - if check_workspace_empty(identity, WORKSPACE_PATH, WORKSPACE_DIR): + if check_workspace_empty(identity, WORKSPACE_PATH, + WORKSPACE_DIR): db_con.db_update(identity, response.WORKSPACE_FAILED) return jsonify(status=response.EMPTY_WORKSPACE) @@ -563,7 +564,8 @@ def linchpin_up(current_user, username) -> Response: else: identity = str(uuid.uuid4()) pinfile_content = data['pinfile_content'] - json_pinfile_path = WORKSPACE_PATH + "/" + identity + PINFILE_JSON_PATH + json_pinfile_path = WORKSPACE_PATH + "/" + identity +\ + PINFILE_JSON_PATH precmd = ["linchpin", "-w " + WORKSPACE_DIR + identity + "/", "init"] output = subprocess.Popen(precmd, stdout=subprocess.PIPE) @@ -580,7 +582,8 @@ def linchpin_up(current_user, username) -> Response: raise ValueError output = subprocess.Popen(cmd, stdout=subprocess.PIPE) output.communicate() - linchpin_latest_path = WORKSPACE_PATH + "/" + identity + LATEST_PATH + linchpin_latest_path = WORKSPACE_PATH + "/" + identity + \ + LATEST_PATH with open(linchpin_latest_path, 'r') as file: linchpin_latest = json.load(file) directory_path = glob.glob(WORKSPACE_PATH + "/" + identity + diff --git a/app/utils/__init__.py b/app/utils/__init__.py index 192176c..173ec59 100644 --- a/app/utils/__init__.py +++ b/app/utils/__init__.py @@ -1,10 +1,7 @@ import os -import json import uuid from app.data_access_layer import RestDB from app.data_access_layer import UserRestDB -from app.response_messages import response -from flask import jsonify from typing import List from werkzeug.security import generate_password_hash From e8c5474f6af581f5b2d74794e594d6f7e7a80ed0 Mon Sep 17 00:00:00 2001 From: Mansi Date: Wed, 21 Aug 2019 11:21:35 -0400 Subject: [PATCH 14/20] added unittests to travis file --- .travis.yml | 1 + app/__init__.py | 8 +++----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6ad9a6d..e1a7c9e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ before_script: cd app script: - flake8 --exclude=\.eggs,tests,docs --ignore=E124,E303,W504 --max-line-length 80 . + - python3 ../tests/UnitTest.py deploy: provider: pypi diff --git a/app/__init__.py b/app/__init__.py index ecd3e86..cc79ce6 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -23,7 +23,7 @@ APP_DIR = os.path.dirname(os.path.realpath(__file__)) try: - with open(APP_DIR + '/config.yml', 'r') as f: + with open(APP_DIR + 'config.yml', 'r') as f: config = yaml.load(f) except Exception as x: config = {} @@ -564,8 +564,7 @@ def linchpin_up(current_user, username) -> Response: else: identity = str(uuid.uuid4()) pinfile_content = data['pinfile_content'] - json_pinfile_path = WORKSPACE_PATH + "/" + identity +\ - PINFILE_JSON_PATH + json_pinfile_path = WORKSPACE_PATH + "/" + identity + PINFILE_JSON_PATH precmd = ["linchpin", "-w " + WORKSPACE_DIR + identity + "/", "init"] output = subprocess.Popen(precmd, stdout=subprocess.PIPE) @@ -582,8 +581,7 @@ def linchpin_up(current_user, username) -> Response: raise ValueError output = subprocess.Popen(cmd, stdout=subprocess.PIPE) output.communicate() - linchpin_latest_path = WORKSPACE_PATH + "/" + identity + \ - LATEST_PATH + linchpin_latest_path = WORKSPACE_PATH + "/" + identity + LATEST_PATH with open(linchpin_latest_path, 'r') as file: linchpin_latest = json.load(file) directory_path = glob.glob(WORKSPACE_PATH + "/" + identity + From f3567117f214c38a0e1f0e84140c4759fbd29b25 Mon Sep 17 00:00:00 2001 From: Mansi Date: Wed, 21 Aug 2019 11:46:34 -0400 Subject: [PATCH 15/20] fixing erros for travis --- .travis.yml | 2 +- app/__init__.py | 17 +++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index e1a7c9e..bc8c395 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ before_script: cd app script: - flake8 --exclude=\.eggs,tests,docs --ignore=E124,E303,W504 --max-line-length 80 . - - python3 ../tests/UnitTest.py + - python3 app/tests/UnitTest.py deploy: provider: pypi diff --git a/app/__init__.py b/app/__init__.py index cc79ce6..a6ecf81 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -14,7 +14,7 @@ from werkzeug.security import generate_password_hash, check_password_hash from flask_swagger_ui import get_swaggerui_blueprint from functools import wraps -from app.utils import get_connection, create_fetch_cmd, create_cmd_workspace,\ +from app.utils import get_connection, create_fetch_cmd, create_cmd_workspace, \ create_cmd_up_pinfile, check_workspace_empty, get_connection_users, \ create_admin_user, check_workspace_has_pinfile @@ -29,7 +29,6 @@ config = {} app.logger.error(x) - # loads defaults when config.yml does not exists or has been removed WORKSPACE_DIR = config.get('workspace_path', '/tmp') LOGGER_FILE = config.get('logger_file_name', 'restylinchpin.log') @@ -83,6 +82,7 @@ def decorated(*args, **kwargs): except Exception as e: return jsonify(message=response.API_KEY_INVALID, status=e) return function(current_user, *args, **kwargs) + return decorated @@ -103,7 +103,7 @@ def new_user(current_user): email = request.json.get('email') api_key = str(uuid.uuid4()) if username is None or password is None: - abort(errors.ERROR_STATUS) # missing arguments + abort(errors.ERROR_STATUS) # missing arguments if db_con.db_get_username(username): return jsonify(message=response.USER_ALREADY_EXISTS) hashed_password = generate_password_hash(password, method='sha256') @@ -357,8 +357,8 @@ def linchpin_init(current_user) -> Response: else: # Checking if workspace name contains any special characters output = subprocess.Popen(["linchpin", "-w " + - WORKSPACE_DIR + identity + - "/", "init"], stdout=subprocess.PIPE) + WORKSPACE_DIR + identity + + "/", "init"], stdout=subprocess.PIPE) db_con.db_update(identity, response.WORKSPACE_SUCCESS) return jsonify(name=data["name"], id=identity, status=response.CREATE_SUCCESS, @@ -433,7 +433,7 @@ def linchpin_delete_workspace(current_user, identity) -> Response: db_con = get_connection(DB_PATH) try: # path specifying location of working directory inside server - workspace_owner_user =\ + workspace_owner_user = \ db_con.db_search_username(current_user['username']) if not current_user['admin'] and not workspace_owner_user: return jsonify(message=response.NOT_FOUND) @@ -564,7 +564,8 @@ def linchpin_up(current_user, username) -> Response: else: identity = str(uuid.uuid4()) pinfile_content = data['pinfile_content'] - json_pinfile_path = WORKSPACE_PATH + "/" + identity + PINFILE_JSON_PATH + json_pinfile_path = \ + WORKSPACE_PATH + "/" + identity + PINFILE_JSON_PATH precmd = ["linchpin", "-w " + WORKSPACE_DIR + identity + "/", "init"] output = subprocess.Popen(precmd, stdout=subprocess.PIPE) @@ -723,7 +724,7 @@ def get_linchpin_latest(current_user, identity) -> Response: else: check_path = "/" linchpin_latest_directory = WORKSPACE_PATH + "/" + identity + check_path - if not os.listdir(linchpin_latest_directory).\ + if not os.listdir(linchpin_latest_directory). \ __contains__(LINCHPIN_LATEST_NAME): return jsonify(message=response.LINCHPIN_LATEST_NOT_FOUND) linchpin_latest_path = linchpin_latest_directory + LINCHPIN_LATEST_NAME From 8cb2f4e3ef1157afc94c9e885224366d9158ced3 Mon Sep 17 00:00:00 2001 From: Mansi Date: Wed, 21 Aug 2019 11:59:25 -0400 Subject: [PATCH 16/20] fixing travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bc8c395..a35ec24 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ before_script: cd app script: - flake8 --exclude=\.eggs,tests,docs --ignore=E124,E303,W504 --max-line-length 80 . - - python3 app/tests/UnitTest.py + - python3 tests/UnitTest.py deploy: provider: pypi From 37021ed5f26457d380e6b0668f34052f4621737e Mon Sep 17 00:00:00 2001 From: Mansi Date: Wed, 21 Aug 2019 12:06:25 -0400 Subject: [PATCH 17/20] fixing cannot identify module error --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a35ec24..cb52d8e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,8 @@ before_script: cd app script: - flake8 --exclude=\.eggs,tests,docs --ignore=E124,E303,W504 --max-line-length 80 . - - python3 tests/UnitTest.py + - cd.. + - python3 app/tests/UnitTest.py deploy: provider: pypi From 9e00e3fb7806a8f583394754a8973238e56ce7d9 Mon Sep 17 00:00:00 2001 From: Mansi Date: Wed, 21 Aug 2019 12:15:17 -0400 Subject: [PATCH 18/20] fixing travis errors --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index cb52d8e..a9b85d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,8 +10,7 @@ before_script: cd app script: - flake8 --exclude=\.eggs,tests,docs --ignore=E124,E303,W504 --max-line-length 80 . - - cd.. - - python3 app/tests/UnitTest.py + - python3 ../app/tests/UnitTest.py deploy: provider: pypi From a5613d81bb2aabb8de59ec43018f74168289912b Mon Sep 17 00:00:00 2001 From: Mansi Date: Wed, 21 Aug 2019 12:22:34 -0400 Subject: [PATCH 19/20] fixing travis errors --- app/tests/UnitTest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tests/UnitTest.py b/app/tests/UnitTest.py index d4bdefb..f905883 100644 --- a/app/tests/UnitTest.py +++ b/app/tests/UnitTest.py @@ -1,5 +1,5 @@ import unittest -import app +from app import app from app.data_access_layer import RestDB from app.data_access_layer import UserRestDB From 06434a054fb2748cb94a64b7c1b193b4cbebee87 Mon Sep 17 00:00:00 2001 From: Mansi Date: Thu, 22 Aug 2019 11:13:23 -0400 Subject: [PATCH 20/20] moved tests outside app --- .travis.yml | 3 ++- {app/tests => tests}/UnitTest.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) rename {app/tests => tests}/UnitTest.py (99%) diff --git a/.travis.yml b/.travis.yml index a9b85d8..fbf80e3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,8 @@ before_script: cd app script: - flake8 --exclude=\.eggs,tests,docs --ignore=E124,E303,W504 --max-line-length 80 . - - python3 ../app/tests/UnitTest.py + - python3 ../tests/UnitTest.py + deploy: provider: pypi diff --git a/app/tests/UnitTest.py b/tests/UnitTest.py similarity index 99% rename from app/tests/UnitTest.py rename to tests/UnitTest.py index f905883..d4bdefb 100644 --- a/app/tests/UnitTest.py +++ b/tests/UnitTest.py @@ -1,5 +1,5 @@ import unittest -from app import app +import app from app.data_access_layer import RestDB from app.data_access_layer import UserRestDB