From d280961f0f7c917d7fa60ab712f3011caa32c9e4 Mon Sep 17 00:00:00 2001 From: Sietse Snel Date: Wed, 24 Apr 2024 08:49:07 +0200 Subject: [PATCH 01/11] README: copy edit --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 60d5178..b25578f 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ Create a symlink to use the contents from the shared folder to replace the msl_a sudo ln -s /media/sf_epos /var/www/msl_api ``` -Check to see if the login page is accessible by navigating to epos-msl.ckan.test/webservice/login, a reboot might be +Check to see if the login page is accessible by navigating to https://epos-msl.ckan.test/webservice/login. A reboot might be needed. ### Seeding test admin panel account(s) @@ -119,9 +119,13 @@ ansible-playbook playbook.yml An API key needs to be generated within CKAN to be used by msl-api for transferring data. -1. Navigate to: https://epos-msl.ckan.test/user/login and sigin in with: +1. Navigate to: https://epos-msl.ckan.test/user/login and sign in in with the following credentials: + +``` Username: ckanadmin Password: testtest +``` + 2. Navigate to: https://epos-msl.ckan.test/user/edit/ckanadmin and click 'Regenerate API key' 3. Copy the API Key displayed in the bottom left 4. Paste the value within the .env file of msl-api for the 'CKAN_API_TOKEN' key From 2ea1e6b87fd6b6992dbbdbcc78b2428837e738af Mon Sep 17 00:00:00 2001 From: Sietse Snel Date: Wed, 24 Apr 2024 16:48:52 +0200 Subject: [PATCH 02/11] Add optional ckanext-spatial plugin For now, it will be disabled by default. --- README.md | 3 ++ environments/development/host_vars/epos-msl | 1 + roles/ckan/defaults/main.yml | 4 +++ roles/ckan/tasks/main.yml | 5 ++++ roles/ckan/tasks/spatial_plugin.yml | 33 +++++++++++++++++++++ roles/ckan/templates/ckan.ini.j2 | 4 +++ 6 files changed, 50 insertions(+) create mode 100644 roles/ckan/tasks/spatial_plugin.yml diff --git a/README.md b/README.md index b25578f..1acf63e 100644 --- a/README.md +++ b/README.md @@ -180,6 +180,9 @@ The main configuration settings are: |msl_api_asset_url | asset URL for the MSL API web service, e.g. https://epos-catalog.mydomain.nl/webservice | |ckan_api_token | the MSL API uses this value to authenticate to the CKAN API. this should currently be the API key (not API token!) of the ckanadmin account. The current way to use this field is: deploy the catalog using a dummy value for this parameter, log in on CKAN using the ckanadmin account, generate an API key, replace the dummy value in the host\_vars file with the real API key, and run the playbook a second time. |msl_api_app_key | the MSL API application key. The current way to configure this is to deploy the application, generate the app key by running `sudo -u www-data /usr/bin/php8.0 artisan key:generate && sudo -u www-data /usr/bin/php8.0 artisan config:cache` in /var/www/msl\_api. Finally copy the generated key in /var/www/msl\_api/.env to the host\_vars file. +| ckan_install_spatial_plugin | whether to install the ckanext-spatial plugin (default: false) | +| ckan_spatial_plugin_repo | Github repository to use for the ckanext-spatial plugin | +| ckan_spatial_plugin_version | Branch or tag to use for the ckanext-spatial plugin | ## CKAN catalog diff --git a/environments/development/host_vars/epos-msl b/environments/development/host_vars/epos-msl index 6cc8dee..7e267fa 100644 --- a/environments/development/host_vars/epos-msl +++ b/environments/development/host_vars/epos-msl @@ -18,6 +18,7 @@ epos_msl_environment: development # EPOS-MSL environment: development, ckan_database_password: test # CKAN Postgresql database password ckan_admin_password: testtest # CKAN admin user ('ckanadmin') password ckan_harvest_password: testtest # CKAN harvest user ('harvest') password +ckan_install_spatial_plugin: true ########################## # MSL API configuration diff --git a/roles/ckan/defaults/main.yml b/roles/ckan/defaults/main.yml index 9ded777..303101f 100644 --- a/roles/ckan/defaults/main.yml +++ b/roles/ckan/defaults/main.yml @@ -43,3 +43,7 @@ ckan_msl_core_plugin_version: 1.4.0 ckan_msl_core_plugin_branch: 1.4.0 ckan_msl_util_plugin_branch: 1.0.0 ckan_msl_vocabularies_endpoint: https://epos-msl.ckan.test/webservice/api/vocabularies + +ckan_install_spatial_plugin: false +ckan_spatial_plugin_repo: ckan/ckanext-spatial +ckan_spatial_plugin_version: v2.1.1 diff --git a/roles/ckan/tasks/main.yml b/roles/ckan/tasks/main.yml index 5d9c9f5..a150d7c 100644 --- a/roles/ckan/tasks/main.yml +++ b/roles/ckan/tasks/main.yml @@ -146,6 +146,11 @@ virtualenv: /usr/lib/ckan/default +- name: Install CKAN spatial plugin, if enabled + ansible.builtin.import_tasks: spatial_plugin.yml + when: ckan_install_spatial_plugin + + - name: Install CKAN scheming plugin ansible.builtin.pip: name: "https://github.com/ckan/ckanext-scheming/archive/{{ ckanext_scheming_version }}.tar.gz" diff --git a/roles/ckan/tasks/spatial_plugin.yml b/roles/ckan/tasks/spatial_plugin.yml new file mode 100644 index 0000000..5b4b74b --- /dev/null +++ b/roles/ckan/tasks/spatial_plugin.yml @@ -0,0 +1,33 @@ +--- +# copyright Utrecht University +# +- name: Install spatial CKAN plugin package dependencies + ansible.builtin.apt: + pkg: + - python-dev + - libxml2-dev + - libxslt1-dev + - libgeos-c1v5 + state: present + + +- name: Clone spatial CKAN plugin + ansible.builtin.git: + repo: "https://github.com/{{ ckan_spatial_plugin_repo }}" + dest: /usr/lib/ckan/ckanext-spatial + version: "{{ ckan_spatial_plugin_version }}" + + +- name: Install spatial CKAN plugin Python dependencies + ansible.builtin.pip: + requirements: /usr/lib/ckan/ckanext-spatial/requirements.txt + virtualenv: /usr/lib/ckan/default + notify: Restart ckan-uwsgi + + +- name: Install spatial CKAN plugin + ansible.builtin.pip: + name: /usr/lib/ckan/ckanext-spatial + virtualenv: /usr/lib/ckan/default + extra_args: "-e" + notify: Restart ckan-uwsgi diff --git a/roles/ckan/templates/ckan.ini.j2 b/roles/ckan/templates/ckan.ini.j2 index 942d416..875358b 100644 --- a/roles/ckan/templates/ckan.ini.j2 +++ b/roles/ckan/templates/ckan.ini.j2 @@ -129,7 +129,11 @@ solr_url = http://127.0.0.1:{{ solr_port }}/solr/ckan # Add ``resource_proxy`` to enable resorce proxying and get around the # same origin policy #ckan.plugins = stats text_view image_view recline_view +{% if ckan_install_spatial_plugin %} +ckan.plugins = stats text_view image_view recline_view msl_ckan scheming_datasets scheming_groups scheming_organizations msl_custom_facets msl_repeating_fields spatial_metadata spatial_query +{% else %} ckan.plugins = stats text_view image_view recline_view msl_ckan scheming_datasets scheming_groups scheming_organizations msl_custom_facets msl_repeating_fields +{% endif %} scheming.dataset_schemas = ckanext.msl_ckan:schemas/datasets/data_publication.yml ckanext.msl_ckan:schemas/datasets/labs.json scheming.organization_schemas = ckanext.msl_ckan:schemas/organizations/organization.json From 037430a42c7d5d0f70bfde24c9846c8095160280 Mon Sep 17 00:00:00 2001 From: Sietse Snel Date: Fri, 26 Apr 2024 13:37:37 +0200 Subject: [PATCH 03/11] CKAN role: add editable option for plugins Add option to allow for installing plugins in editable mode, which is more convenient for testing and development. This is now enabled for the development environment, but otherwise disabled by default. --- README.md | 1 + environments/development/host_vars/epos-msl | 1 + roles/ckan/defaults/main.yml | 1 + roles/ckan/tasks/main.yml | 45 +++------------------ roles/ckan/tasks/msl_core_plugin.yml | 37 +++++++++++++++++ roles/ckan/tasks/msl_util_plugin.yml | 27 +++++++++++++ roles/ckan/tasks/scheming_plugin.yml | 27 +++++++++++++ roles/ckan/tasks/spatial_plugin.yml | 12 +++++- 8 files changed, 110 insertions(+), 41 deletions(-) create mode 100644 roles/ckan/tasks/msl_core_plugin.yml create mode 100644 roles/ckan/tasks/msl_util_plugin.yml create mode 100644 roles/ckan/tasks/scheming_plugin.yml diff --git a/README.md b/README.md index 1acf63e..8cc750f 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,7 @@ The main configuration settings are: | ckan_install_spatial_plugin | whether to install the ckanext-spatial plugin (default: false) | | ckan_spatial_plugin_repo | Github repository to use for the ckanext-spatial plugin | | ckan_spatial_plugin_version | Branch or tag to use for the ckanext-spatial plugin | +| ckan_plugins_editable_mode | Whether to install CKAN plugins in editable mode. This is convenient for development and testing purposes. Enabled on development environment; default value is `false`. | ## CKAN catalog diff --git a/environments/development/host_vars/epos-msl b/environments/development/host_vars/epos-msl index 7e267fa..6adc90b 100644 --- a/environments/development/host_vars/epos-msl +++ b/environments/development/host_vars/epos-msl @@ -19,6 +19,7 @@ ckan_database_password: test # CKAN Postgresql database password ckan_admin_password: testtest # CKAN admin user ('ckanadmin') password ckan_harvest_password: testtest # CKAN harvest user ('harvest') password ckan_install_spatial_plugin: true +ckan_plugins_editable_mode: true ########################## # MSL API configuration diff --git a/roles/ckan/defaults/main.yml b/roles/ckan/defaults/main.yml index 303101f..2257045 100644 --- a/roles/ckan/defaults/main.yml +++ b/roles/ckan/defaults/main.yml @@ -24,6 +24,7 @@ ckan_session_timeout: 3600 ckan_package_url: https://packaging.ckan.org/python-ckan_2.9-py3-focal_amd64.deb ckan_package_file: /tmp/ckan_package.deb ckan_package_upgrade: false +ckan_plugins_editable_mode: false ckan_ini_file: /etc/ckan/default/ckan.ini ckan_storage_path: /ckandata ckan_search_facets_limit: 10000 diff --git a/roles/ckan/tasks/main.yml b/roles/ckan/tasks/main.yml index a150d7c..1327c0d 100644 --- a/roles/ckan/tasks/main.yml +++ b/roles/ckan/tasks/main.yml @@ -152,34 +152,15 @@ - name: Install CKAN scheming plugin - ansible.builtin.pip: - name: "https://github.com/ckan/ckanext-scheming/archive/{{ ckanext_scheming_version }}.tar.gz" - virtualenv: /usr/lib/ckan/default - extra_args: "--upgrade" - notify: Restart ckan-uwsgi - - -- name: Install CKAN MSL Core plugin - ansible.builtin.pip: - name: "git+https://github.com/UtrechtUniversity/msl_ckan_core@{{ ckan_msl_core_plugin_branch }}" - virtualenv: /usr/lib/ckan/default - extra_args: "--upgrade" - notify: Restart ckan-uwsgi + ansible.builtin.import_tasks: scheming_plugin.yml -- name: Clone CKAN MSL Core plugin for copying images - ansible.builtin.git: - repo: "https://github.com/UtrechtUniversity/msl_ckan_core" - dest: /usr/lib/ckan/msl_ckan_core - version: "{{ ckan_msl_core_plugin_branch }}" +- name: Install MSL Core plugin + ansible.builtin.import_tasks: msl_core_plugin.yml -# Workaround for issue where pip doesn't install static images in MSL core plugin -- name: Synchronize MSL Core plugin images - ansible.posix.synchronize: - src: /usr/lib/ckan/msl_ckan_core/ckanext/msl_ckan/public/ - dest: /usr/lib/ckan/default/lib/python3.8/site-packages/ckanext/msl_ckan/public/ - delegate_to: "{{ inventory_hostname }}" +- name: Install MSL util plugin + ansible.builtin.import_tasks: msl_util_plugin.yml - name: Set Solr schema directory for native Solr package @@ -252,22 +233,6 @@ state: link -# Workaround for issue where pip doesn't install static images in MSL core plugin -- name: Synchronize MSL Core plugin images - ansible.posix.synchronize: - src: /usr/lib/ckan/msl_ckan_core/ckanext/msl_ckan/public/ - dest: /usr/lib/ckan/default/lib/python3.8/site-packages/ckanext/msl_ckan/public/ - delegate_to: "{{ inventory_hostname }}" - - -- name: Install CKAN MSL Util plugin - ansible.builtin.pip: - name: "git+https://github.com/UtrechtUniversity/msl_ckan_util@{{ ckan_msl_util_plugin_branch }}" - virtualenv: /usr/lib/ckan/default - extra_args: "--upgrade" - notify: Restart ckan-uwsgi - - - name: Check CKAN database initialized become: true become_user: postgres diff --git a/roles/ckan/tasks/msl_core_plugin.yml b/roles/ckan/tasks/msl_core_plugin.yml new file mode 100644 index 0000000..df4917b --- /dev/null +++ b/roles/ckan/tasks/msl_core_plugin.yml @@ -0,0 +1,37 @@ +--- +# copyright Utrecht University + +# It is always cloned for copying images, and depending on settings, also for an +# editable install of the plugin. +- name: Clone CKAN MSL Core plugin + ansible.builtin.git: + repo: "https://github.com/UtrechtUniversity/msl_ckan_core" + dest: /usr/lib/ckan/msl_ckan_core + version: "{{ ckan_msl_core_plugin_branch }}" + + +- name: Install CKAN MSL Core plugin (regular mode) + ansible.builtin.pip: + name: "git+https://github.com/UtrechtUniversity/msl_ckan_core@{{ ckan_msl_core_plugin_branch }}" + virtualenv: /usr/lib/ckan/default + extra_args: "--upgrade" + notify: Restart ckan-uwsgi + when: not ckan_plugins_editable_mode + + +- name: Install CKAN MSL Core plugin (editable mode) + ansible.builtin.pip: + name: /usr/lib/ckan/msl_ckan_core + virtualenv: /usr/lib/ckan/default + extra_args: "-e" + when: ckan_plugins_editable_mode + notify: Restart ckan-uwsgi + + +# Workaround for issue where pip doesn't install static images in MSL core plugin +- name: Synchronize MSL Core plugin images + ansible.posix.synchronize: + src: /usr/lib/ckan/msl_ckan_core/ckanext/msl_ckan/public/ + dest: /usr/lib/ckan/default/lib/python3.8/site-packages/ckanext/msl_ckan/public/ + delegate_to: "{{ inventory_hostname }}" + when: not ckan_plugins_editable_mode diff --git a/roles/ckan/tasks/msl_util_plugin.yml b/roles/ckan/tasks/msl_util_plugin.yml new file mode 100644 index 0000000..22e1217 --- /dev/null +++ b/roles/ckan/tasks/msl_util_plugin.yml @@ -0,0 +1,27 @@ +--- +# copyright Utrecht University + +- name: Clone CKAN MSL util plugin + ansible.builtin.git: + repo: "https://github.com/utrechtuniversity/msl_ckan_util" + dest: /usr/lib/ckan/msl_ckan_util + version: "{{ ckan_msl_util_plugin_branch }}" + when: ckan_plugins_editable_mode + notify: Restart ckan-uwsgi + + +- name: Install CKAN MSL Util plugin + ansible.builtin.pip: + name: "git+https://github.com/UtrechtUniversity/msl_ckan_util@{{ ckan_msl_util_plugin_branch }}" + virtualenv: /usr/lib/ckan/default + extra_args: "--upgrade" + notify: Restart ckan-uwsgi + + +- name: Install CKAN MSL Util plugin (editable mode) + ansible.builtin.pip: + name: /usr/lib/ckan/msl_ckan_util + virtualenv: /usr/lib/ckan/default + extra_args: "-e" + when: ckan_plugins_editable_mode + notify: Restart ckan-uwsgi diff --git a/roles/ckan/tasks/scheming_plugin.yml b/roles/ckan/tasks/scheming_plugin.yml new file mode 100644 index 0000000..6babedd --- /dev/null +++ b/roles/ckan/tasks/scheming_plugin.yml @@ -0,0 +1,27 @@ +--- +# copyright Utrecht University + +- name: Clone scheming CKAN plugin + ansible.builtin.git: + repo: "https://github.com/ckan/ckanext-scheming" + dest: /usr/lib/ckan/ckanext-scheming + version: "{{ ckanext_scheming_version }}" + notify: Restart ckan-uwsgi + + +- name: Install CKAN scheming plugin (regular mode) + ansible.builtin.pip: + name: "https://github.com/ckan/ckanext-scheming/archive/{{ ckanext_scheming_version }}.tar.gz" + virtualenv: /usr/lib/ckan/default + extra_args: "--upgrade" + when: not ckan_plugins_editable_mode + notify: Restart ckan-uwsgi + + +- name: Install CKAN scheming plugin (editable mode) + ansible.builtin.pip: + name: /usr/lib/ckan/ckanext-scheming + virtualenv: /usr/lib/ckan/default + extra_args: "-e" + when: ckan_plugins_editable_mode + notify: Restart ckan-uwsgi diff --git a/roles/ckan/tasks/spatial_plugin.yml b/roles/ckan/tasks/spatial_plugin.yml index 5b4b74b..8392037 100644 --- a/roles/ckan/tasks/spatial_plugin.yml +++ b/roles/ckan/tasks/spatial_plugin.yml @@ -25,9 +25,19 @@ notify: Restart ckan-uwsgi -- name: Install spatial CKAN plugin +- name: Install spatial CKAN plugin (regular mode) + ansible.builtin.pip: + name: "git+https://github.com/{{ ckan_spatial_plugin_repo }}@{{ ckan_spatial_plugin_version }}" + virtualenv: /usr/lib/ckan/default + extra_args: "--upgrade" + when: not ckan_plugins_editable_mode + notify: Restart ckan-uwsgi + + +- name: Install spatial CKAN plugin (editable mode) ansible.builtin.pip: name: /usr/lib/ckan/ckanext-spatial virtualenv: /usr/lib/ckan/default extra_args: "-e" + when: ckan_plugins_editable_mode notify: Restart ckan-uwsgi From e76cabbeed1aa9cb25d077dc583f8192adfc396d Mon Sep 17 00:00:00 2001 From: Sietse Snel Date: Fri, 26 Apr 2024 13:56:37 +0200 Subject: [PATCH 04/11] CKAN role: remove api.php template No longer used in current version for the catalog --- roles/ckan/templates/api.php.j2 | 164 -------------------------------- 1 file changed, 164 deletions(-) delete mode 100644 roles/ckan/templates/api.php.j2 diff --git a/roles/ckan/templates/api.php.j2 b/roles/ckan/templates/api.php.j2 deleted file mode 100644 index 3b64585..0000000 --- a/roles/ckan/templates/api.php.j2 +++ /dev/null @@ -1,164 +0,0 @@ - 'author', - 'Keywords' => 'tags', - 'Title' => 'title', - 'Description' => 'notes', - 'Lab' => 'organization', - 'Dataset_contact' => 'extras_Datasetcontact', - 'References' => 'extras_References', - 'Is_Supplement_To' => 'extras_Issupplementto', - 'Cites' => 'extras_Cites', - 'Publisher' => 'extras_Publisher', - 'PublicationYear' => 'extras_Publicationdate' -); - -$solrQuery = []; -foreach ($querystring as $param => $value) { - if (array_key_exists($param, $fieldMapping)) { - $param = $fieldMapping[$param]; - - //str_replace(' ', '%20', $value) - - $solrQuery[] = sprintf( - '(%s:"%s")', - $param, - $value - ); - } -} - -$api_url = sprintf( - 'https://localhost/api/3/action/package_search?q=%s', - implode(' AND ', $solrQuery) -); - -$s = file_get_contents(str_replace(' ', '%20', $api_url)); - -$response = json_decode($s, JSON_OBJECT_AS_ARRAY); -$countDPs = 0; -$totalDPs = array(); - -if ($response['success'] == 1){ - if ($response['result']['count']>0) { - $results = $response['result']['results']; - - foreach ($results as $result) { - $tags = array(); - foreach ($result['tags'] as $tag) { - $tags[] = $tag['name']; - } - $subDomains = array(); - foreach ($result['groups'] as $subDomain) { - $subDomains = $subDomain['display_name']; - } - - //geoJSON - bounding box - $bbX1 = 0; - $bbY1 = 0; - $bbX2 = 0; - $bbY2 = 0; - - $dpData = array( - 'Title' => $result['title'], - 'Description' => $result['notes'], - 'Author' => $result['author'], - 'Keywords' => $tags, - 'Subdomain' => $subDomains, - 'Lab' => $result['organization']['title'], - - 'Publisher' => $result['maintainer'], // provided by - 'Publisher_email' => $result['maintainer_email'], // - 'Url' => $result['url'], - 'License' => $result['license_title'], - ); - - foreach ($result['extras'] as $extra) { //pairs are enumerated - if (substr($extra['key'],0,7)!='harvest') { - if (substr($extra['key'],0,6)=='geobox') { - // Fill the coordinates with floats - $val = floatval($extra['value']); - switch ($extra['key']) { - case 'geobox-eLong': - $bbX1 = $val; - break; - case 'geobox-nLat': - $bbY1 = $val; - break; - case 'geobox-wLong': - $bbX2 = $val; - break; - case 'geobox-sLat': - $bbY2 = $val; - } - } - else { - $dpData[$extra['key']] = $extra['value']; - } - } - } - - $geoJSON = array( - 'type' => 'Feature', - 'properties' => $dpData, - 'geometry' => array( - 'type' => 'Polygon', - 'coordinates' => array( - array( - array($bbX1,$bbY1), array($bbX2,$bbY1), array($bbX2,$bbY2), array($bbX1,$bbY2),array($bbX1,$bbY1) - ) - ) - ) - ); - - $totalDPs[] = $geoJSON; - $countDPs += 1; - } - } -} - -echo json_encode($totalDPs); -exit; From d5d751dceec2554fb419cf64cdb1b5c911e34b6b Mon Sep 17 00:00:00 2001 From: Samsh001 Date: Fri, 26 Apr 2024 14:42:07 +0200 Subject: [PATCH 05/11] fix reboot command in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8cc750f..0102502 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ sudo adduser vagrant vboxsf Next, restart the server: ```bash -sudo restart +sudo reboot ``` After rebooting ssh into epos-msl again and the content of the share should be visible within /media/sf_epos! From 176fbb38933c2aa9d698d0b17a6632f146013f8c Mon Sep 17 00:00:00 2001 From: Samsh001 Date: Fri, 26 Apr 2024 21:45:18 +0200 Subject: [PATCH 06/11] update README explaining local development with CKAN extensions --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index 0102502..1eb4a49 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,21 @@ sudo ln -s /media/sf_epos /var/www/msl_api Check to see if the login page is accessible by navigating to https://epos-msl.ckan.test/webservice/login. A reboot might be needed. +#### CKAN extensions + +The same method can be used to work with local folders containing the msl CKAN extensions by creating shared folders +within the Virtualbox management program as described above. For example after creating a shared folder containing +the 'msl_ckan_core' repo checkout can be used within the epos-msl server by first removing the currently present version +on the server with: +```bash +sudo rm -r /usr/lib/ckan/msl_ckan_core +``` + +Followed by creating a symlink to the shared mount: +```bash +sudo ln -s /media/sf_ckanext-msl_ckan /usr/lib/ckan/msl_ckan_core +``` + ### Seeding test admin panel account(s) The msl_api project contains a specific seeder for adding test admin accounts. Contents can be adjusted to add or From d852a66148e455aa48aaed4bb59566ee1d6c7348 Mon Sep 17 00:00:00 2001 From: Sietse Snel Date: Wed, 1 May 2024 15:59:33 +0200 Subject: [PATCH 07/11] Development env: install new Solr version Use upstream Solr version (which is much newer than the distribution version) by default on development environment, since we want to use this version as a basis for future development. --- environments/development/host_vars/epos-msl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/environments/development/host_vars/epos-msl b/environments/development/host_vars/epos-msl index 6adc90b..cf89357 100644 --- a/environments/development/host_vars/epos-msl +++ b/environments/development/host_vars/epos-msl @@ -30,3 +30,9 @@ msl_api_app_url: http://epos-msl.ckan.test/webservice msl_api_asset_url: https://epos-msl.ckan.test/webservice msl_api_app_key: "" ckan_api_token: none + +######################### +# Solr configuration +######################### + +solr_install_native: false From 287acb95c90047f1d00e63ec294637a5f87f9257 Mon Sep 17 00:00:00 2001 From: Sietse Snel Date: Wed, 1 May 2024 20:28:41 +0200 Subject: [PATCH 08/11] Solr role: use upstream Solr by default now --- roles/solr/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/solr/defaults/main.yml b/roles/solr/defaults/main.yml index 8622c01..54904a7 100644 --- a/roles/solr/defaults/main.yml +++ b/roles/solr/defaults/main.yml @@ -3,7 +3,7 @@ # This installs an old version of Solr that comes with a native Ubuntu package, # rather than a version from Solr.org -solr_install_native: true +solr_install_native: false # CKAN 2.9 only supports Solr 7 and 8. This playbook uses the Solr 8 CKAN schema, # so it only works with Solr 8.x The settings below only take effect when solr_install_native From 9fe153b2df3910ae701b1a3e1d3d898be5aef4ea Mon Sep 17 00:00:00 2001 From: Samsh001 Date: Sat, 4 May 2024 13:10:31 +0200 Subject: [PATCH 09/11] Add how to connect to the mysql database from host machine to README. --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 1eb4a49..1878270 100644 --- a/README.md +++ b/README.md @@ -181,6 +181,16 @@ sudo -u www-data /usr/bin/php8.0 artisan db:seed sudo -u www-data /usr/bin/php8.0 artisan storage:link ``` +## Connecting to the mysql database from local machine + +To connect from the host machine to the mysql database used by msl_api use the following command: + +``` +vagrant ssh epos-msl -- -L 3306:127.0.0.1:3306 -N epos-msl +``` + +While running, you can connect to the database using the default credentials. + ## Configuration The main configuration settings are: From 35139a9d54d6bae4188cc7d10f776edface05c37 Mon Sep 17 00:00:00 2001 From: Sietse Snel Date: Tue, 8 Oct 2024 10:52:48 +0200 Subject: [PATCH 10/11] YDA-5949: Add Docker setup for development --- .ansible-lint | 3 + docker/.env | 48 + docker/README.md | 28 + docker/build-local-images.sh | 13 + docker/docker-compose.yml | 102 ++ docker/images/ckan/Dockerfile | 106 ++ docker/images/ckan/build.sh | 2 + docker/images/ckan/ckan-entrypoint.sh | 65 + docker/images/ckan/ckan-uwsgi.ini | 15 + docker/images/ckan/ckan.ini | 274 ++++ docker/images/ckan/who.ini | 32 + docker/images/ckan/wsgi.py | 12 + docker/images/msl-api/Dockerfile | 64 + docker/images/msl-api/build.sh | 2 + docker/images/msl-api/msl-api-entrypoint.sh | 74 + docker/images/msl-api/msl-api.env | 55 + docker/images/msl-api/msl-api.site | 34 + docker/images/nginx/Dockerfile | 11 + docker/images/nginx/build.sh | 2 + docker/images/nginx/ckan.site | 68 + docker/images/nginx/epos-msl.cnf | 13 + docker/images/nginx/nginx-entrypoint.sh | 24 + docker/images/solr/Dockerfile | 27 + docker/images/solr/build.sh | 2 + docker/images/solr/solrconfig.xml | 1360 +++++++++++++++++++ 25 files changed, 2436 insertions(+) create mode 100644 docker/.env create mode 100644 docker/README.md create mode 100755 docker/build-local-images.sh create mode 100644 docker/docker-compose.yml create mode 100644 docker/images/ckan/Dockerfile create mode 100755 docker/images/ckan/build.sh create mode 100644 docker/images/ckan/ckan-entrypoint.sh create mode 100644 docker/images/ckan/ckan-uwsgi.ini create mode 100644 docker/images/ckan/ckan.ini create mode 100644 docker/images/ckan/who.ini create mode 100644 docker/images/ckan/wsgi.py create mode 100644 docker/images/msl-api/Dockerfile create mode 100755 docker/images/msl-api/build.sh create mode 100644 docker/images/msl-api/msl-api-entrypoint.sh create mode 100644 docker/images/msl-api/msl-api.env create mode 100644 docker/images/msl-api/msl-api.site create mode 100644 docker/images/nginx/Dockerfile create mode 100755 docker/images/nginx/build.sh create mode 100644 docker/images/nginx/ckan.site create mode 100644 docker/images/nginx/epos-msl.cnf create mode 100644 docker/images/nginx/nginx-entrypoint.sh create mode 100644 docker/images/solr/Dockerfile create mode 100755 docker/images/solr/build.sh create mode 100644 docker/images/solr/solrconfig.xml diff --git a/.ansible-lint b/.ansible-lint index 27fb9d4..8d34551 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -1,3 +1,6 @@ +exclude_paths: + - ./.github + - ./docker skip_list: - no-changed-when # Commands should not change things if nothing needs doing - no-handler # Tasks that run when changed should likely be handlers diff --git a/docker/.env b/docker/.env new file mode 100644 index 0000000..3d4ee5e --- /dev/null +++ b/docker/.env @@ -0,0 +1,48 @@ +# Variables in this file will be substituted into docker-compose.yml +# Save a copy of this file as .env and insert your own values. +# Verify correct substitution with "docker-compose config" +# If variables are newly added or enabled, please delete and rebuild the images to pull in changes: +# docker-compose down +# docker rmi -f docker_ckan docker_db +# docker rmi $(docker images -f dangling=true -q) +# docker-compose build +# docker-compose up -d +# docker-compose restart ckan # give the db service time to initialize the db cluster on first run + +# Image: ckan +CKAN_SITE_ID=default + +EPOS_MSL_HOST=epos-msl.ckan +# +# On AWS, your CKAN_SITE_URL is the output of: +# curl -s http://169.254.169.254/latest/meta-data/public-hostname +# CKAN_SITE_URL=http://ec2-xxx-xxx-xxx-xxx.ap-southeast-2.compute.amazonaws.com +# On OSX with Docker for Mac, your CKAN_SITE_URL is +# CKAN_SITE_URL=http://docker.for.mac.localhost:5000 +# When running locally, CKAN_SITE_URL must contain the port +CKAN_SITE_URL=http://localhost:5000 +# +# CKAN_PORT must be available on the host: sudo netstat -na +# To apply change: docker-compose down && docker rmi docker_ckan && docker-compose build ckan +CKAN_PORT=5000 +# +# Email settings +CKAN_SMTP_SERVER=smtp.corporateict.domain:25 +CKAN_SMTP_STARTTLS=True +CKAN_SMTP_USER=user +CKAN_SMTP_PASSWORD=pass +CKAN_SMTP_MAIL_FROM=ckan@localhost +# +# Image: db +POSTGRES_PASSWORD=ckan +# +# POSTGRES_PORT must be available on the host: sudo netstat -na | grep 5432 +# To apply change: docker-compose down && docker rmi docker_db docker_ckan && docker-compose build +POSTGRES_PORT=5432 +# +# The datastore database will be created in the db container as docs +# Readwrite user/pass will be ckan:POSTGRES_PASSWORD +# Readonly user/pass will be datastore_ro:DATASTORE_READONLY_PASSWORD +DATASTORE_READONLY_PASSWORD=datastore + +MYSQL_ROOT_PASSWORD=testtest diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..dacdbb6 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,28 @@ +# EPOS-MSL Docker development setup + +This Docker setup currently has an experimental status, and is in development. + +## Building the images + +The images of the EPOS-MSL are not yet available in a registry, so you'll have to build them locally first. + +``` +./build-local-images.sh +``` + +## Using the Docker setup + +First add an entry to your `/etc/hosts` file (or equivalent) so that queries for the development setup +interface resolve to your loopback interface. For example: + +``` +127.0.0.1 epos-msl.ckan +``` + +Start the Docker Compose setup: +``` +docker compose up +``` + +Wait until CKAN has started. Then navigate to [https://epos-msl.ckan](https://epos-msl.ckan) in your browser. The +development VM runs with self-signed certificates, so you'll need to accept the security warning. diff --git a/docker/build-local-images.sh b/docker/build-local-images.sh new file mode 100755 index 0000000..0d72b48 --- /dev/null +++ b/docker/build-local-images.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -e + +cd images + +for image in ckan nginx solr msl-api +do cd "$image" + echo "Building image $image ..." + ./build.sh + cd .. +done + +echo "Building images completed." diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..cb89915 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,102 @@ +# This is a locally modified version of the upstream CKAN docker compose file. +# It contains image names for pushing locally built images to a central registry at ECR + +# docker-compose build && docker-compose up -d +# If "docker-compose logs ckan" shows DB not ready, run "docker-compose restart ckan" a few times. +version: "3" + +volumes: + ckan_config: + ckan_home: + ckan_storage: + ckan_coveragedata: + pg_data: + solr_data: + nginx_config: + nginx_certificates: + mslapi_signal: + mslapi_storage: + +services: + ckan: + container_name: ckan + image: epos-msl-ckan:latest + links: + - db + - solr + - redis + depends_on: + - db + - solr + - redis + ports: + - "0.0.0.0:${CKAN_PORT}:5000" + environment: + - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan_default + - CKAN_SOLR_URL=http://solr:8983/solr/ckan + - CKAN_REDIS_URL=redis://redis:6379/1 + - CKAN_SITE_URL=${CKAN_SITE_URL} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + volumes: + - ckan_config:/etc/ckan + - ckan_home:/usr/lib/ckan + - ckan_storage:/var/lib/ckan + - ckan_coveragedata:/coverage + + nginx: + container_name: nginx + image: epos-msl-nginx:latest + environment: + - EPOS_MSL_HOST=${EPOS_MSL_HOST} + ports: + - "18443:443" + volumes: + - nginx_config:/etc/nginx/conf.d + - nginx_certificates:/etc/certificates + + db: + container_name: db + image: postgres:12.20 + environment: + - POSTGRES_DB=ckan_default + - POSTGRES_USER=ckan + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - PGDATA=/var/lib/postgresql/data/db + volumes: + - pg_data:/var/lib/postgresql/data + + mslapi_db: + container_name: mslapi_db + image: mysql:8.0.39-debian + environment: + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + + mslapi_webserver: + container_name: mslapi_web + image: epos-msl-api:latest + environment: + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + - MSLAPI_ROLE=WEBSERVER + volumes: + - mslapi_signal:/signal + - mslapi_storage:/storage + + mslapi_worker: + container_name: mslapi_worker + image: epos-msl-api:latest + environment: + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + - MSLAPI_ROLE=QUEUE_WORKER + volumes: + - mslapi_signal:/signal + - mslapi_storage:/storage + + solr: + container_name: solr + image: epos-msl-solr:latest + volumes: + - solr_data:/opt/solr/server/solr/ckan/data + + redis: + container_name: redis + image: redis:6.2 diff --git a/docker/images/ckan/Dockerfile b/docker/images/ckan/Dockerfile new file mode 100644 index 0000000..de6d49e --- /dev/null +++ b/docker/images/ckan/Dockerfile @@ -0,0 +1,106 @@ +# This is a locally modified version of the upstream CKAN Dockerfile + +# See CKAN docs on installation from Docker Compose on usage +FROM ubuntu:focal-20210119 +MAINTAINER Yoda team + +# Set timezone +ENV TZ=UTC +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# Setting the locale +ENV LC_ALL=en_US.UTF-8 +RUN adduser ckan +RUN apt-get update +RUN apt-get install --no-install-recommends -y locales +RUN sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen +RUN dpkg-reconfigure --frontend=noninteractive locales +RUN update-locale LANG=${LC_ALL} + +# Install required system packages +RUN apt-get -q -y update \ + && DEBIAN_FRONTEND=noninteractive apt-get -q -y upgrade \ + && apt-get -q -y install \ + python3.8 \ + python3-dev \ + python3-pip \ + python3-venv \ + python3-wheel \ + libpq-dev \ + python3-pastescript \ + python3-virtualenv \ + libxml2-dev \ + libxslt-dev \ + libgeos-dev \ + libssl-dev \ + libffi-dev \ + postgresql-client \ + build-essential \ + git-core \ + vim \ + wget \ + curl \ + nmap \ + sqlite3 \ + pwgen \ + uuid-runtime \ + && apt-get -q clean \ + && rm -rf /var/lib/apt/lists/* + +# Define environment variables +ENV CKAN_HOME /usr/lib/ckan +ENV CKAN_VENV $CKAN_HOME/default +ENV CKAN_CONFIG /etc/ckan +ENV CKAN_STORAGE_PATH=/ckanstorage +ENV CKAN_VERSION=2.9.11 +ENV CKAN_SCHEMING_VERSION=release-2.1.0 +ENV CKAN_MSL_CORE_VERSION=1.4.0 +ENV CKAN_MSL_UTIL_VERSION=1.0.0 + +# Create storage path +RUN mkdir -p $CKAN_STORAGE_PATH/webassets $CKAN_STORAGE_PATH/storage + +# Build-time variables specified by docker-compose.yml / .env +ARG CKAN_SITE_URL + +# Setup virtual environment for CKAN +RUN mkdir -p $CKAN_VENV $CKAN_CONFIG/default && \ + python3 -m venv $CKAN_VENV && \ + ln -s $CKAN_VENV/bin/pip3 /usr/local/bin/ckan-pip3 &&\ + ln -s $CKAN_VENV/bin/ckan /usr/local/bin/ckan +ADD ckan.ini /etc/ckan/default/ckan.ini +ADD who.ini /etc/ckan/default/who.ini +ADD wsgi.py /etc/ckan/default/wsgi.py +ADD ckan-uwsgi.ini /etc/ckan/default/ckan-uwsgi.ini + +# Virtual environment binaries/scripts to be used first +ENV PATH=${CKAN_VENV}/bin:${PATH} +ENV EPOS_MSL_FQDN=epos-msl.local +ENV CKAN_ADMIN_PASSWORD="testtest" + +# Install CKAN and plugins +RUN ckan-pip3 install -U pip && \ + ckan-pip3 install setuptools==44.1.0 && \ + ckan-pip3 install --upgrade pip && \ + ckan-pip3 install wheel && \ + ckan-pip3 install -e "git+https://github.com/ckan/ckan@ckan-${CKAN_VERSION}#egg=ckan[requirements]" && \ + ckan-pip3 install uwsgi && \ + ckan-pip3 install -e "git+https://github.com/ckan/ckanext-scheming@${CKAN_SCHEMING_VERSION}#egg=ckanext-scheming" && \ + ckan-pip3 install -e "git+https://github.com/UtrechtUniversity/msl_ckan_core@${CKAN_MSL_CORE_VERSION}#egg=ckanext-msl_ckan" && \ + ckan-pip3 install -e "git+https://github.com/UtrechtUniversity/msl_ckan_util@${CKAN_MSL_UTIL_VERSION}#egg=ckanext-msl_ckan_util" && \ + ln -s $CKAN_VENV/src/ckan/ckan/config/who.ini $CKAN_CONFIG/who.ini && \ + cp -v $CKAN_VENV/src/ckan/contrib/docker/ckan-entrypoint.sh /ckan-entrypoint.sh && \ + chmod +x /ckan-entrypoint.sh && \ + chown -R ckan:ckan $CKAN_HOME $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH $COVERAGE_DIR && \ + rm /usr/lib/ckan/default/src/ckan/ckan/config/solr/schema.xml && \ + ln -sf /usr/lib/ckan/default/src/ckanext-msl-ckan/ckanext/msl_ckan/config/solr/schema.xml /usr/lib/ckan/default/src/ckan/ckan/config/solr/schema.xml && \ + perl -n -i.bak -e 'print unless /defaultSearchField/ or /solrQueryParser/' /usr/lib/ckan/default/src/ckan/ckan/config/solr/schema.xml + +ADD ./ckan-entrypoint.sh /ckan-entrypoint.sh +ENTRYPOINT ["/ckan-entrypoint.sh"] +RUN chmod +x /ckan-entrypoint.sh + +USER ckan +EXPOSE 8080 + +CMD ["/ckan-entrypoint.sh"] diff --git a/docker/images/ckan/build.sh b/docker/images/ckan/build.sh new file mode 100755 index 0000000..9712778 --- /dev/null +++ b/docker/images/ckan/build.sh @@ -0,0 +1,2 @@ +#!/bin/sh +docker build -t epos-msl-ckan:latest . diff --git a/docker/images/ckan/ckan-entrypoint.sh b/docker/images/ckan/ckan-entrypoint.sh new file mode 100644 index 0000000..53dfa4e --- /dev/null +++ b/docker/images/ckan/ckan-entrypoint.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +set -u + +check_port() { + local host="$1" + local port="$2" + + echo "Checking service availability at ${host}:${port}..." + + while true; do + # Use nmap to scan the specified port on the given host + nmap -p "${port}" "${host}" | grep -q "open" + + # If the port is open, exit the loop + if [ $? -eq 0 ]; then + echo "Server is available at ${host}:${port}" + break + else + echo "Server at ${host}:${port} is not available. Retrying in 1 seconds..." + sleep 1 + fi + done +} + +## Check DB up +check_port db 5432 + +## Check Solr up +check_port solr 8983 + +## Check Redis up +check_port redis 6379 + +## Initialize CKAN config, database and admin account +CKAN_CONFIG_FILE=/etc/ckan/default/ckan.ini +CKAN_INIT_STATUS_FILE=/etc/ckan/default/.ckan_initialized + +if test -f "$CKAN_INIT_STATUS_FILE" +then echo "Configuration and database already initialized." +else echo "Initializing configuration ..." + export BEAKER_SESSION_SECRET=$(openssl rand -base64 32) + export APP_INSTANCE_UUID=$(uuidgen --name "$EPOS_MSL_FQDN" --namespace "@url" --sha1) + export CKAN_DATABASE_PASSWORD=$(pwgen -n 16 -N 1) + export CKAN_MSL_VOCABULARIES_ENDPOINT="https://${EPOS_MSL_FQDN}/webservice/api/vocabularies" + perl -pi.bak -e '$beaker_session_secret=$ENV{BEAKER_SESSION_SECRET}; s/BEAKER_SESSION_SECRET/$beaker_session_secret/ge' "$CKAN_CONFIG_FILE" + perl -pi.bak -e '$app_instance_uuid=$ENV{APP_INSTANCE_UUID}; s/APP_INSTANCE_UUID/$app_instance_uuid/ge' "$CKAN_CONFIG_FILE" + perl -pi.bak -e '$ckan_database_password=$ENV{CKAN_DATABASE_PASSWORD}; s/CKAN_DATABASE_PASSWORD/$ckan_database_password/ge' "$CKAN_CONFIG_FILE" + perl -pi.bak -e '$epos_msl_fqdn=$ENV{EPOS_MSL_FQDN}; s/EPOS_MSL_FQDN/$epos_msl_fqdn/ge' "$CKAN_CONFIG_FILE" + perl -pi.bak -e '$ckan_msl_vocabularies_endpoint=$ENV{CKAN_MSL_VOCABULARIES_ENDPOINT}; s/CKAN_MSL_VOCABULARIES_ENDPOINT/$ckan_msl_vocabularies_endpoint/ge' "$CKAN_CONFIG_FILE" + echo "Initializing database ..." + /usr/lib/ckan/default/bin/ckan -c "$CKAN_CONFIG_FILE" db init + /usr/lib/ckan/default/bin/ckan -c "$CKAN_CONFIG_FILE" user add ckanadmin password="$CKAN_ADMIN_PASSWORD" email=ckanadmin@localhost name=ckanadmin + /usr/lib/ckan/default/bin/ckan -c "$CKAN_CONFIG_FILE" sysadmin add ckanadmin + touch "$CKAN_INIT_STATUS_FILE" + echo "Configuration and database initialization finished." +fi + +## Start CKAN +echo "Starting CKAN ..." +while true +do /usr/lib/ckan/default/bin/uwsgi -i /etc/ckan/default/ckan-uwsgi.ini + sleep 1 + echo "CKAN process terminated; trying to restart ..." +done diff --git a/docker/images/ckan/ckan-uwsgi.ini b/docker/images/ckan/ckan-uwsgi.ini new file mode 100644 index 0000000..d30aa91 --- /dev/null +++ b/docker/images/ckan/ckan-uwsgi.ini @@ -0,0 +1,15 @@ +[uwsgi] + +http = 0.0.0.0:8080 +uid = www-data +gid = www-data +wsgi-file = /etc/ckan/default/wsgi.py +virtualenv = /usr/lib/ckan/default +module = wsgi:application +master = true +pidfile = /tmp/%n.pid +harakiri = 50 +max-requests = 5000 +vacuum = true +callable = application +strict = true diff --git a/docker/images/ckan/ckan.ini b/docker/images/ckan/ckan.ini new file mode 100644 index 0000000..c1f591b --- /dev/null +++ b/docker/images/ckan/ckan.ini @@ -0,0 +1,274 @@ +# +# CKAN - Pylons configuration +# +# These are some of the configuration options available for your CKAN +# instance. Check the documentation in 'doc/configuration.rst' or at the +# following URL for a description of what they do and the full list of +# available options: +# +# http://docs.ckan.org/en/latest/maintaining/configuration.html +# +# The %(here)s variable will be replaced with the parent directory of this file +# + +[DEFAULT] + +# WARNING: *THIS SETTING MUST BE SET TO FALSE ON A PUBLIC ENVIRONMENT* +# With debug mode enabled, a visitor to your site could execute malicious commands. +debug = false + +[app:main] +use = egg:ckan + +## Development settings +ckan.devserver.host = localhost +ckan.devserver.port = 5000 + +## Session settings +cache_dir = /tmp/%(ckan.site_id)s/ +beaker.session.key = ckan + +# This is the secret token that the beaker library uses to hash the cookie sent +# to the client. `ckan generate config` generates a unique value for this each +# time it generates a config file. +beaker.session.secret = BEAKER_SESSION_SECRET + +# `ckan generate config` generates a unique value for this each time it generates +# a config file. +app_instance_uuid = APP_INSTANCE_UUID + +# Default number of search facets returned in a query +search.facets.limit = 10000 + +# repoze.who config +who.config_file = %(here)s/who.ini +who.log_level = warning +who.log_file = %(cache_dir)s/who_log.ini +# Session timeout (user logged out after period of inactivity, in seconds). +# Inactive by default, so the session doesn't expire. +who.timeout = 3600 + +## Database Settings +sqlalchemy.url = postgresql://ckan:CKAN_DATABASE_PASSWORD@db/ckan_default + +#ckan.datastore.write_url = postgresql://ckan_default:pass@localhost/datastore_default +#ckan.datastore.read_url = postgresql://datastore_default:pass@localhost/datastore_default + +# PostgreSQL' full-text search parameters +ckan.datastore.default_fts_lang = english +ckan.datastore.default_fts_index_method = gist + + +## Site Settings + +ckan.site_url = https://EPOS_MSL_FQDN +#ckan.use_pylons_response_cleanup_middleware = true + +# Default timeout for Requests +#ckan.requests.timeout = 10 + + +## Authorization Settings + +ckan.auth.anon_create_dataset = false +ckan.auth.create_unowned_dataset = false +ckan.auth.create_dataset_if_not_in_organization = false +ckan.auth.user_create_groups = false +ckan.auth.user_create_organizations = false +ckan.auth.user_delete_groups = true +ckan.auth.user_delete_organizations = true +ckan.auth.create_user_via_api = false +ckan.auth.create_user_via_web = false +ckan.auth.roles_that_cascade_to_sub_groups = admin +ckan.auth.public_user_details = false +ckan.auth.public_activity_stream_detail = false +ckan.auth.allow_dataset_collaborators = false +ckan.auth.create_default_api_keys = false + +## API Token Settings +api_token.nbytes = 60 +api_token.jwt.encode.secret = string:HvlXPoquK7PcVZ1eMo7Lwf_Rs +api_token.jwt.decode.secret = string:HvlXPoquK7PcVZ1eMo7Lwf_Rs +api_token.jwt.algorithm = HS256 + +## API Token: expire_api_token plugin +expire_api_token.default_lifetime = 3600 + +## Search Settings + +ckan.site_id = default + +solr_url = http://solr:8983/solr/ckan + +## Redis Settings + +# URL to your Redis instance, including the database to be used. +ckan.redis.url = redis://redis:6379/0 + + +## CORS Settings + +# If cors.origin_allow_all is true, all origins are allowed. +# If false, the cors.origin_whitelist is used. +# ckan.cors.origin_allow_all = true +# cors.origin_whitelist is a space separated list of allowed domains. +# ckan.cors.origin_whitelist = http://example1.com http://example2.com + + +## Plugins Settings + +# Note: Add ``datastore`` to enable the CKAN DataStore +# Add ``datapusher`` to enable DataPusher +# Add ``resource_proxy`` to enable resorce proxying and get around the +# same origin policy +#ckan.plugins = stats text_view image_view recline_view +ckan.plugins = stats text_view image_view recline_view msl_ckan scheming_datasets scheming_groups scheming_organizations msl_custom_facets msl_repeating_fields + +scheming.dataset_schemas = ckanext.msl_ckan:schemas/datasets/data_publication.yml ckanext.msl_ckan:schemas/datasets/labs.json +scheming.organization_schemas = ckanext.msl_ckan:schemas/organizations/organization.json + +mslfacets.dataset_config = ckanext.msl_ckan:config/facets.json + +mslindexfields.field_config = ckanext.msl_ckan:config/msl_index_fields.json + +mslvocabularies.endpoint_root = CKAN_MSL_VOCABULARIES_ENDPOINT + +# Define which views should be created by default +# (plugins must be loaded in ckan.plugins) +ckan.views.default_views = image_view text_view recline_view + +# Customize which text formats the text_view plugin will show +#ckan.preview.json_formats = json +#ckan.preview.xml_formats = xml rdf rdf+xml owl+xml atom rss +#ckan.preview.text_formats = text plain text/plain + +# Customize which image formats the image_view plugin will show +#ckan.preview.image_formats = png jpeg jpg gif + +## Front-End Settings + +ckan.site_title = CKAN +ckan.site_logo = /base/images/ckan-logo.png +ckan.site_description = +ckan.favicon = /base/images/ckan.ico +ckan.gravatar_default = identicon +ckan.preview.direct = png jpg gif +ckan.preview.loadable = html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json +ckan.display_timezone = server + +# package_hide_extras = for_search_index_only +#package_edit_return_url = http://another.frontend/dataset/ +#package_new_return_url = http://another.frontend/dataset/ +#ckan.recaptcha.publickey = +#ckan.recaptcha.privatekey = +#licenses_group_url = http://licenses.opendefinition.org/licenses/groups/ckan.json +# ckan.template_footer_end = + + +## Internationalisation Settings +ckan.locale_default = en +ckan.locale_order = en pt_BR ja it cs_CZ ca es fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv +ckan.locales_offered = +ckan.locales_filtered_out = en_GB + +## Feeds Settings + +ckan.feeds.authority_name = +ckan.feeds.date = +ckan.feeds.author_name = +ckan.feeds.author_link = + +## Storage Settings + +ckan.storage_path = /ckanstorage +#ckan.max_resource_size = 10 +#ckan.max_image_size = 2 + +## Webassets Settings +#ckan.webassets.use_x_sendfile = false +#ckan.webassets.path = /var/lib/ckan/webassets + + +## Datapusher settings + +# Make sure you have set up the DataStore + +#ckan.datapusher.formats = csv xls xlsx tsv application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet +#ckan.datapusher.url = http://127.0.0.1:8800/ +#ckan.datapusher.assume_task_stale_after = 3600 + +# Resource Proxy settings +# Preview size limit, default: 1MB +#ckan.resource_proxy.max_file_size = 1048576 +# Size of chunks to read/write. +#ckan.resource_proxy.chunk_size = 4096 +# Default timeout for fetching proxied items +#ckan.resource_proxy.timeout = 10 + +## Activity Streams Settings + +#ckan.activity_streams_enabled = true +#ckan.activity_list_limit = 31 +#ckan.activity_streams_email_notifications = true +#ckan.email_notifications_since = 2 days +ckan.hide_activity_from_users = %(ckan.site_id)s + + +## Email settings + +#email_to = errors@example.com +#error_email_from = ckan-errors@example.com +#smtp.server = localhost +#smtp.starttls = False +#smtp.user = username@example.com +#smtp.password = your_password +#smtp.mail_from = +#smtp.reply_to = + +## Background Job Settings +ckan.jobs.timeout = 180 + +## Logging configuration +[loggers] +keys = root, ckan, ckanext, werkzeug + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console + +[logger_werkzeug] +level = WARNING +handlers = console +qualname = werkzeug +propagate = 0 + +[logger_ckan] +level = INFO +handlers = console +qualname = ckan +propagate = 0 + +[logger_ckanext] +level = DEBUG +handlers = console +qualname = ckanext +propagate = 0 + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s + +# msl_ckan_util settings +ckan.mslfacets.dataset_config = ckanext.msl_ckan_util:samples/facets.json +ckan.mslindexfields.fields_config = ckanext.msl_ckan_util:samples/msl_index_fields.json diff --git a/docker/images/ckan/who.ini b/docker/images/ckan/who.ini new file mode 100644 index 0000000..eb2a5a5 --- /dev/null +++ b/docker/images/ckan/who.ini @@ -0,0 +1,32 @@ +[plugin:auth_tkt] +use = ckan.lib.repoze_plugins.auth_tkt:make_plugin +# If no secret key is defined here, beaker.session.secret will be used +#secret = somesecret + +[plugin:friendlyform] +use = ckan.lib.repoze_plugins.friendly_form:FriendlyFormPlugin +login_form_url= /user/login +login_handler_path = /login_generic +logout_handler_path = /user/logout +rememberer_name = auth_tkt +post_login_url = /user/logged_in +post_logout_url = /user/logged_out +charset = utf-8 + +[general] +request_classifier = repoze.who.classifiers:default_request_classifier +challenge_decider = repoze.who.classifiers:default_challenge_decider + +[identifiers] +plugins = + friendlyform;browser + auth_tkt + +[authenticators] +plugins = + auth_tkt + ckan.lib.authenticator:UsernamePasswordAuthenticator + +[challengers] +plugins = + friendlyform;browser diff --git a/docker/images/ckan/wsgi.py b/docker/images/ckan/wsgi.py new file mode 100644 index 0000000..9880af9 --- /dev/null +++ b/docker/images/ckan/wsgi.py @@ -0,0 +1,12 @@ +# -- coding: utf-8 -- + +import os +from ckan.config.middleware import make_app +from ckan.cli import CKANConfigLoader +from logging.config import fileConfig as loggingFileConfig +config_filepath = os.path.join( + os.path.dirname(os.path.abspath(__file__)), 'ckan.ini') +abspath = os.path.join(os.path.dirname(os.path.abspath(__file__))) +loggingFileConfig(config_filepath) +config = CKANConfigLoader(config_filepath).get_config() +application = make_app(config) diff --git a/docker/images/msl-api/Dockerfile b/docker/images/msl-api/Dockerfile new file mode 100644 index 0000000..1028e7b --- /dev/null +++ b/docker/images/msl-api/Dockerfile @@ -0,0 +1,64 @@ +# This is an image for running the MSL-API application, that is part of the EPOS-MSL catalog + +# See CKAN docs on installation from Docker Compose on usage +FROM ubuntu:focal-20210119 +MAINTAINER Yoda team + +# Set timezone +ENV TZ=UTC +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# Setting the locale +ENV LC_ALL=en_US.UTF-8 +RUN apt-get update +RUN apt-get install --no-install-recommends -y locales +RUN sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen +RUN dpkg-reconfigure --frontend=noninteractive locales +RUN update-locale LANG=${LC_ALL} + +# Install required system packages +ENV COMPOSER_VERSION=2.5.4 +RUN apt-get -q -y update \ + && DEBIAN_FRONTEND=noninteractive apt-get -q -y upgrade \ + && apt-get -q -y install \ + nginx \ + acl \ + git \ + nginx \ + wget \ + nmap \ + sudo \ + software-properties-common \ + && apt-add-repository ppa:ondrej/php \ + && apt-get -q -y install \ + php8.0 \ + php8.0-bcmath \ + php8.0-common \ + php8.0-curl \ + php8.0-fpm \ + php8.0-gd \ + php8.0-mbstring \ + php8.0-mysql \ + php8.0-xml \ + php8.0-zip \ + mysql-client \ + && apt-get -q clean \ + && rm -rf /var/lib/apt/lists/* \ + && wget "https://getcomposer.org/download/$COMPOSER_VERSION/composer.phar" \ + -O /usr/local/bin/composer2 + +ADD msl-api.site /etc/nginx/conf.d/msl-api.conf + +ADD msl-api-entrypoint.sh /msl-api-entrypoint.sh +RUN chmod +x /msl-api-entrypoint.sh + +RUN cd /var/www && \ + git clone https://github.com/utrechtUniversity/msl_api && \ + chown -R www-data:www-data msl_api +USER www-data +ADD msl-api.env /var/www/msl_api/.env +RUN cd /var/www/msl_api && \ + /usr/bin/php8.0 /usr/local/bin/composer2 install + +USER root +CMD ["/msl-api-entrypoint.sh"] diff --git a/docker/images/msl-api/build.sh b/docker/images/msl-api/build.sh new file mode 100755 index 0000000..0485a35 --- /dev/null +++ b/docker/images/msl-api/build.sh @@ -0,0 +1,2 @@ +#!/bin/sh +docker build -t epos-msl-api:latest . diff --git a/docker/images/msl-api/msl-api-entrypoint.sh b/docker/images/msl-api/msl-api-entrypoint.sh new file mode 100644 index 0000000..98151b8 --- /dev/null +++ b/docker/images/msl-api/msl-api-entrypoint.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +set -u + +check_port() { + local host="$1" + local port="$2" + + echo "Checking service availability at ${host}:${port}..." + + while true; do + # Use nmap to scan the specified port on the given host + nmap -p "${port}" "${host}" | grep -q "open" + + # If the port is open, exit the loop + if [ $? -eq 0 ]; then + echo "Server is available at ${host}:${port}" + break + else + echo "Server at ${host}:${port} is not available. Retrying in 1 seconds..." + sleep 1 + fi + done +} + +## Check DB up +check_port mslapi_db 3306 + +# The web server container initializes the database and application +# The queue worker container waits for the web server to finish initialization +SIGNALFILE="/signal/mslapi_initialized.sig" +if [ -f "$SIGNALFILE" ] +then echo "MSL-API already initialized. Proceeding with starting $MSLAPI_ROLE ..." +else + if [ "$MSLAPI_ROLE" == "WEBSERVER" ] + then # Initialize the MSL-API database + mysql -u root "-p$MYSQL_ROOT_PASSWORD" -h mslapi_db -e " +CREATE DATABASE mslapi; +CREATE USER 'msl'@'%' IDENTIFIED BY 'msl'; +GRANT ALL PRIVILEGES ON mslapi.* TO 'msl'@'%'; +FLUSH PRIVILEGES; +" + cd /var/www/msl_api + # Initialize the MSL-API application + set -x + sudo -u www-data /usr/bin/php8.0 artisan key:generate + sudo -u www-data /usr/bin/php8.0 artisan config:cache + sudo -u www-data /usr/bin/php8.0 artisan migrate --force + sudo -u www-data /usr/bin/php8.0 artisan db:seed --force + sudo -u www-data /usr/bin/php8.0 artisan storage:link + set +x + touch "$SIGNALFILE" + elif [ "$MSLAPI_ROLE" == "QUEUE_WORKER" ] + then while ! [ -f "$SIGNALFILE" ] + do echo "Waiting for web server to initialize MSL-API application ..." + sleep 2 + done + echo "MSL-API application initialized. Proceeding ..." + else echo "Error: unknown MSL API role: $MSLAPI_ROLE" + exit 1 + fi +fi + +## Run main process +if [ "$MSLAPI_ROLE" == "QUEUE_WORKER" ] +then while true + do sudo -u www-data /usr/bin/php8.0 /var/www/msl_api/artisan queue:work --rest=1 --tries=3 --timeout=300 + sleep 3 + echo "Restarting queue worker after exit..." + done +elif [ "$MSLAPI_ROLE" == "WEBSERVER" ] +then /usr/sbin/nginx -g 'daemon off;' +else echo "Error: unknown MSL API role: $MSLAPI_ROLE" +fi diff --git a/docker/images/msl-api/msl-api.env b/docker/images/msl-api/msl-api.env new file mode 100644 index 0000000..ae2e0c9 --- /dev/null +++ b/docker/images/msl-api/msl-api.env @@ -0,0 +1,55 @@ +APP_NAME=MSLAPI +APP_ENV=production +APP_DEBUG=false +APP_URL=https://epos-msl.ckan/webservice +ASSET_URL=https://epos-msl.ckan/webservice + +LOG_CHANNEL=stack +LOG_LEVEL=debug + +DB_CONNECTION=mysql +DB_HOST=mslapi_db +DB_PORT=3306 +DB_DATABASE=mslapi +DB_USERNAME=msl +DB_PASSWORD=msl + +BROADCAST_DRIVER=log +CACHE_DRIVER=file +FILESYSTEM_DRIVER=local +QUEUE_CONNECTION=database +SESSION_DRIVER=file +SESSION_LIFETIME=120 + +MEMCACHED_HOST=127.0.0.1 + +REDIS_HOST=127.0.0.1 +REDIS_PASSWORD=null +REDIS_PORT=6379 + +MAIL_MAILER=smtp +MAIL_HOST=mailhog +MAIL_PORT=1025 +MAIL_USERNAME=null +MAIL_PASSWORD=null +MAIL_ENCRYPTION=null +MAIL_FROM_ADDRESS=null +MAIL_FROM_NAME="${APP_NAME}" + +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +AWS_DEFAULT_REGION=us-east-1 +AWS_BUCKET= +AWS_USE_PATH_STYLE_ENDPOINT=false + +PUSHER_APP_ID= +PUSHER_APP_KEY= +PUSHER_APP_SECRET= +PUSHER_APP_CLUSTER=mt1 + +MIX_PUSHER_APP_KEY="${PUSHER_APP_KEY}" +MIX_PUSHER_APP_CLUSTER="${PUSHER_APP_CLUSTER}" + +CKAN_API_URL= https://ckan/api/3/ +CKAN_API_TOKEN= PUT_API_TOKEN_HERE +CKAN_ROOT_URL= https://epos-msl.ckan/ diff --git a/docker/images/msl-api/msl-api.site b/docker/images/msl-api/msl-api.site new file mode 100644 index 0000000..1bb1cdc --- /dev/null +++ b/docker/images/msl-api/msl-api.site @@ -0,0 +1,34 @@ +proxy_cache_path /tmp/nginx_cache levels=1:2 keys_zone=cache:30m max_size=250m; +proxy_temp_path /tmp/nginx_proxy 1 2; + +server { + listen 80 default_server; + listen [::]:80 default_server; + + location / { + # This server only serves MSL-API traffic + return 404; + } + + location /webservice/ { + alias /var/www/msl_api/public; + try_files /index.php =404; + autoindex on; + + fastcgi_split_path_info ^(.+\.php)(/.+)$; + fastcgi_pass unix:/run/php/php8.0-fpm.sock; + fastcgi_index index.php; + + include fastcgi_params; + fastcgi_param SCRIPT_FILENAME $document_root/index.php; + } + + location /webservice/css/ { + alias /var/www/msl_api/public/css/; + } + + location /webservice/js/ { + alias /var/www/msl_api/public/js/; + } +} + diff --git a/docker/images/nginx/Dockerfile b/docker/images/nginx/Dockerfile new file mode 100644 index 0000000..13ccae5 --- /dev/null +++ b/docker/images/nginx/Dockerfile @@ -0,0 +1,11 @@ +# Image for Nginx reverse proxy configured for CKAN + +FROM nginx:1.27 +MAINTAINER Yoda team + +ADD ckan.site /etc/nginx/conf.d/ckan.conf +RUN rm /etc/nginx/conf.d/default.conf +ADD epos-msl.cnf /etc/certificates/epos-msl.cnf +ADD nginx-entrypoint.sh /nginx-entrypoint.sh +RUN chmod +x /nginx-entrypoint.sh +CMD ["/nginx-entrypoint.sh"] diff --git a/docker/images/nginx/build.sh b/docker/images/nginx/build.sh new file mode 100755 index 0000000..4c015cc --- /dev/null +++ b/docker/images/nginx/build.sh @@ -0,0 +1,2 @@ +#!/bin/sh +docker build -t epos-msl-nginx:latest . diff --git a/docker/images/nginx/ckan.site b/docker/images/nginx/ckan.site new file mode 100644 index 0000000..1e9b5fe --- /dev/null +++ b/docker/images/nginx/ckan.site @@ -0,0 +1,68 @@ +proxy_cache_path /tmp/nginx_cache levels=1:2 keys_zone=cache:30m max_size=250m; +proxy_temp_path /tmp/nginx_proxy 1 2; + +server { + listen 80 default_server; + listen [::]:80 default_server; + + location / { + return 301 https://$host$request_uri; + } +} + +server { + + listen 443 ssl http2; + listen [::]:443 ssl http2; + + ssl_certificate /etc/certificates/epos-msl.pem; + ssl_certificate_key /etc/certificates/epos-msl.key; + ssl_session_timeout 1d; + ssl_session_cache shared:MozSSL:10m; # about 40000 sessions + ssl_session_tickets off; + + # curl https://ssl-config.mozilla.org/ffdhe2048.txt > /path/to/dhparam + ssl_dhparam /etc/certificates/dhparams.pem; + + # intermediate configuration + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers off; + + # HSTS (ngx_http_headers_module is required) (63072000 seconds) + add_header Strict-Transport-Security "max-age=63072000" always; + + # OCSP stapling + ssl_stapling on; + ssl_stapling_verify on; + + # verify chain of trust of OCSP response using Root CA and Intermediate certs + ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt; + + client_max_body_size 100M; + + location / { + proxy_pass http://ckan:8080/; + proxy_set_header Host $host; + proxy_cache cache; + proxy_cache_bypass $cookie_auth_tkt; + proxy_no_cache $cookie_auth_tkt; + proxy_cache_valid 30m; + proxy_cache_key $host$scheme$proxy_host$request_uri; + # In emergency comment out line to force caching + # proxy_ignore_headers X-Accel-Expires Expires Cache-Control; + } + + location /webservice/ { + proxy_pass http://mslapi_webserver:80/; + proxy_set_header Host $host; + proxy_cache cache; + proxy_cache_bypass $cookie_auth_tkt; + proxy_no_cache $cookie_auth_tkt; + proxy_cache_valid 30m; + proxy_cache_key $host$scheme$proxy_host$request_uri; + # In emergency comment out line to force caching + # proxy_ignore_headers X-Accel-Expires Expires Cache-Control; + } + +} diff --git a/docker/images/nginx/epos-msl.cnf b/docker/images/nginx/epos-msl.cnf new file mode 100644 index 0000000..6cd1532 --- /dev/null +++ b/docker/images/nginx/epos-msl.cnf @@ -0,0 +1,13 @@ +[ req ] +default_bits = 4096 +default_md = sha256 +prompt = no +encrypt_key = no +distinguished_name = dn + +[ dn ] +CN = "EPOS_MSL_HOST" +C = NL +O = Yoda +OU = ITS +L = Utrecht diff --git a/docker/images/nginx/nginx-entrypoint.sh b/docker/images/nginx/nginx-entrypoint.sh new file mode 100644 index 0000000..8043edd --- /dev/null +++ b/docker/images/nginx/nginx-entrypoint.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +## Adapt URL in config +export EPOS_MSL_HOST="$EPOS_MSL_HOST" +perl -pi.bak -e '$epos_msl_host=$ENV{EPOS_MSL_HOST}; s/EPOS_MSL_HOST/$epos_msl_host/ge' /etc/certificates/epos-msl.cnf + +## Generate certificates if needed +cd /etc/certificates +if [ -f "epos-msl.pem" ] +then echo "Skipping certificate generation, because certificate files are already present." +else echo "Generating certificates for reverse proxy at https://$EPOS_MSL_HOST ..." + openssl req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes -keyout epos-msl.key -out epos-msl.pem -config epos-msl.cnf + echo "Certificate generation complete." +fi + +if [ -f "dhparam.pem" ] +then echo "Skipping DHParam generation, because DHParam file is already present." +else echo "Generating DHParam configuration..." + openssl dhparam -dsaparam -out dhparams.pem 4096 + echo "DHParam generation complete." +fi + +## Run Nginx +nginx -g "daemon off;" diff --git a/docker/images/solr/Dockerfile b/docker/images/solr/Dockerfile new file mode 100644 index 0000000..e9028da --- /dev/null +++ b/docker/images/solr/Dockerfile @@ -0,0 +1,27 @@ +# Image for Solr configured for CKAN + +FROM solr:8.11 +MAINTAINER Yoda team + +ENV SOLR_CONFIG_DIR="/opt/solr/server/solr/configsets" + +# Create a CKAN configset based on the default one +USER root +RUN cp -R $SOLR_CONFIG_DIR/_default $SOLR_CONFIG_DIR/ckan + +# Download the EPOS-MSL core plugin (for the schema) +RUN cd /usr/lib && \ + apt update && \ + apt install -y git && \ + git clone https://github.com/UtrechtUniversity/msl_ckan_core.git + +# Change the CKAN configset to use the EPOS-MSL schema, and enable Query Elevation +USER root +ADD solrconfig.xml /opt/solr-8.11.4/server/solr/configsets/ckan/conf/solrconfig.xml +RUN ln -s /usr/lib/msl_ckan_core/ckanext/msl_ckan/config/solr/schema.xml $SOLR_CONFIG_DIR/ckan/conf/schema.xml && \ + perl -n -i.bak -e 'print unless /defaultSearchField/ or /solrQueryParser/' $SOLR_CONFIG_DIR/ckan/conf/schema.xml && \ + cp $SOLR_CONFIG_DIR/ckan/conf/schema.xml $SOLR_CONFIG_DIR/ckan/conf/managed-schema && \ + cp /opt/solr/server/solr/configsets/sample_techproducts_configs/conf/elevate.xml $SOLR_CONFIG_DIR/ckan/conf/elevate.xml + +USER solr +CMD ["sh", "-c", "solr-precreate ckan $SOLR_CONFIG_DIR/ckan"] diff --git a/docker/images/solr/build.sh b/docker/images/solr/build.sh new file mode 100755 index 0000000..957e4cd --- /dev/null +++ b/docker/images/solr/build.sh @@ -0,0 +1,2 @@ +#!/bin/sh +docker build -t epos-msl-solr:latest . diff --git a/docker/images/solr/solrconfig.xml b/docker/images/solr/solrconfig.xml new file mode 100644 index 0000000..43edf9b --- /dev/null +++ b/docker/images/solr/solrconfig.xml @@ -0,0 +1,1360 @@ + + + + + + + + 6.6.3 + + + + + + + + + + + + + + + + + + + + ${solr.data.dir:} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.lock.type:native} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.ulog.dir:} + ${solr.ulog.numVersionBuckets:65536} + + + + + ${solr.autoCommit.maxTime:15000} + false + + + + + + ${solr.autoSoftCommit.maxTime:-1} + + + + + + + + + + + + + 1024 + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + 20 + + + 200 + + + + + + + + + + + + + + + + false + + + + + + + + + + + + + + + + + + + + + + + AND + explicit + 10 + + + + + + + + + + + + + + + explicit + json + true + + + + + + + + explicit + + + + + + _text_ + + + + + + + true + ignored_ + _text_ + + + + + + + + + text_general + + + + + + default + _text_ + solr.DirectSolrSpellChecker + + internal + + 0.5 + + 2 + + 1 + + 5 + + 4 + + 0.01 + + + + + + + + + + + + default + on + true + 10 + 5 + 5 + true + true + 10 + 5 + + + spellcheck + + + + + + + + + + true + + + tvComponent + + + + + + + + + + + + true + false + + + terms + + + + + + + + string + elevate.xml + + + + + + explicit + + + elevator + + + + + + + + + + + 100 + + + + + + + + 70 + + 0.5 + + [-\w ,/\n\"']{20,200} + + + + + + + ]]> + ]]> + + + + + + + + + + + + + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + 10 + .,!? + + + + + + + WORD + + + en + US + + + + + + + + + + + + + + [^\w-\.] + _ + + + + + + + yyyy-MM-dd'T'HH:mm:ss.SSSZ + yyyy-MM-dd'T'HH:mm:ss,SSSZ + yyyy-MM-dd'T'HH:mm:ss.SSS + yyyy-MM-dd'T'HH:mm:ss,SSS + yyyy-MM-dd'T'HH:mm:ssZ + yyyy-MM-dd'T'HH:mm:ss + yyyy-MM-dd'T'HH:mmZ + yyyy-MM-dd'T'HH:mm + yyyy-MM-dd HH:mm:ss.SSSZ + yyyy-MM-dd HH:mm:ss,SSSZ + yyyy-MM-dd HH:mm:ss.SSS + yyyy-MM-dd HH:mm:ss,SSS + yyyy-MM-dd HH:mm:ssZ + yyyy-MM-dd HH:mm:ss + yyyy-MM-dd HH:mmZ + yyyy-MM-dd HH:mm + yyyy-MM-dd + + + + + + + + + + + + + + + + + + + + + + + text/plain; charset=UTF-8 + + + + + ${velocity.template.base.dir:} + ${velocity.solr.resource.loader.enabled:true} + ${velocity.params.resource.loader.enabled:false} + + + + + 5 + + + + + + + + + + + + + + + From df1e294aec4e64e5765449e587e5c891519e4019 Mon Sep 17 00:00:00 2001 From: Sietse Snel Date: Thu, 24 Oct 2024 17:18:41 +0200 Subject: [PATCH 11/11] Docker setup: add README note about autocrlf --- docker/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/README.md b/docker/README.md index dacdbb6..221698e 100644 --- a/docker/README.md +++ b/docker/README.md @@ -2,6 +2,10 @@ This Docker setup currently has an experimental status, and is in development. +If you use Windows, ensure that core.autocrlf is set to false in your git client before you clone the EPOS-MSL +repository: _git config --global core.autocrlf false_ Otherwise the Docker images may not work due to line +ending changes. + ## Building the images The images of the EPOS-MSL are not yet available in a registry, so you'll have to build them locally first.