deposits: create ImportData component, validator

cap/config.py

@@ -764,7 +764,7 @@ def _(x):
         },
         'files_serializers': {
             'application/json': (
-                'cap.modules.deposit.serializers:files_response'
+                'invenio_deposit.serializers:json_file_response'
             ),
         },
         'search_class': 'cap.modules.deposit.search:CAPDepositSearch',

cap/modules/deposit/__init__.py

@@ -25,5 +25,3 @@
 """CAP Deposit module."""
 
 from __future__ import absolute_import, print_function
-
-from .cli import create_deposit  # noqa

cap/modules/deposit/api.py

@@ -53,10 +53,32 @@
 from sqlalchemy.orm.exc import NoResultFound
 from werkzeug.local import LocalProxy
 
-from cap.modules.deposit.errors import DisconnectWebhookError, FileUploadError
+from cap.modules.deposit.errors import (
+    DepositValidationError,
+    DisconnectWebhookError,
+    FileUploadError,
+    ReviewError,
+    UniqueRequiredValidationError,
+    UpdateDepositPermissionsError,
+)
+from cap.modules.deposit.fetchers import cap_deposit_fetcher
 from cap.modules.deposit.loaders import get_val_from_path
+from cap.modules.deposit.minters import cap_deposit_minter
+from cap.modules.deposit.permissions import (
+    AdminDepositPermission,
+    CloneDepositPermission,
+    DepositAdminActionNeed,
+    DepositReadActionNeed,
+    DepositUpdateActionNeed,
+    ReviewDepositPermission,
+    UpdateDepositPermission,
+)
+from cap.modules.deposit.review import Reviewable
 from cap.modules.deposit.utils import perform_copying_fields
-from cap.modules.deposit.validators import NoRequiredValidator
+from cap.modules.deposit.validators import (
+    NoRequiredValidator,
+    get_custom_validator,
+)
 from cap.modules.experiments.permissions import exp_need_factory
 from cap.modules.records.api import CAPRecord
 from cap.modules.records.errors import get_error_path
@@ -77,26 +99,6 @@
     get_existing_or_register_user,
 )
 
-from .errors import (
-    DepositValidationError,
-    ReviewError,
-    UniqueRequiredValidationError,
-    UpdateDepositPermissionsError,
-)
-from .fetchers import cap_deposit_fetcher
-from .minters import cap_deposit_minter
-from .permissions import (
-    AdminDepositPermission,
-    CloneDepositPermission,
-    DepositAdminActionNeed,
-    DepositReadActionNeed,
-    DepositUpdateActionNeed,
-    ReviewDepositPermission,
-    UpdateDepositPermission,
-)
-from .review import Reviewable
-from .validators import get_custom_validator
-
 _datastore = LocalProxy(lambda: current_app.extensions["security"].datastore)
 
 PRESERVE_FIELDS = (

cap/modules/deposit/cli.py

@@ -25,9 +25,9 @@
 
 from __future__ import absolute_import, print_function
 
-import os
 import copy
 import json
+import os
 import uuid
 from datetime import datetime
 
@@ -42,13 +42,16 @@
 from cap.modules.deposit.api import CAPDeposit
 from cap.modules.deposit.fetchers import cap_deposit_fetcher
 from cap.modules.deposit.minters import cap_deposit_minter
+from cap.modules.deposit.utils import add_read_permission_for_egroup
 from cap.modules.fixtures.cli import fixtures
-from cap.modules.user.utils import get_existing_or_register_user, \
-    get_existing_or_register_role
-from cap.modules.schemas.resolvers import resolve_schema_by_url, \
-    schema_name_to_url
-
-from .utils import add_read_permission_for_egroup
+from cap.modules.schemas.resolvers import (
+    resolve_schema_by_url,
+    schema_name_to_url,
+)
+from cap.modules.user.utils import (
+    get_existing_or_register_role,
+    get_existing_or_register_user,
+)
 
 
 @fixtures.command('add')
@@ -80,7 +83,8 @@ def add(file_path, schema, version, egroup, usermail, limit):
 
                 click.secho(
                     'Draft with id {} already exist!'.format(pid_value),
-                    fg='red')
+                    fg='red',
+                )
 
             except PIDDoesNotExistError:
                 record_uuid = uuid.uuid4()
@@ -91,29 +95,45 @@ def add(file_path, schema, version, egroup, usermail, limit):
                 if egroup:
                     add_read_permission_for_egroup(deposit, egroup)
 
-                click.secho('Draft {} added.'.format(pid.pid_value),
-                            fg='green')
+                click.secho('Draft {} added.'.format(pid.pid_value), fg='green')
 
         db.session.commit()
 
 
 @fixtures.command('create-deposit')
-@click.option('--file', '-f',
-              type=click.Path(exists=True),
-              required=True,
-              help='JSON data file')
-@click.option('--ana', '-a',
-              help='Type of analysis',)
-@click.option('--role', '-r',
-              'roles', multiple=True,
-              help='Role with access to the record')
-@click.option('--user', '-u',
-              'users', multiple=True,
-              help='User with access to the record')
-@click.option('--owner', '-o',
-              help='Owner of the record')
-@click.option('--save-errors-to', '-e', 'save_errors',
-              help="Provide a filename, that wrong records will be saved to.")
+@click.option(
+    '--file',
+    '-f',
+    type=click.Path(exists=True),
+    required=True,
+    help='JSON data file',
+)
+@click.option(
+    '--ana',
+    '-a',
+    help='Type of analysis',
+)
+@click.option(
+    '--role',
+    '-r',
+    'roles',
+    multiple=True,
+    help='Role with access to the record',
+)
+@click.option(
+    '--user',
+    '-u',
+    'users',
+    multiple=True,
+    help='User with access to the record',
+)
+@click.option('--owner', '-o', help='Owner of the record')
+@click.option(
+    '--save-errors-to',
+    '-e',
+    'save_errors',
+    help="Provide a filename, that wrong records will be saved to.",
+)
 @with_appcontext
 def create_deposit(file, ana, roles, users, owner, save_errors):
     """Create a new deposit through the CLI.
@@ -146,8 +166,7 @@ def save_errors_to_json(save_errors, errors):
     """Saves the wrong records to a specified file."""
     timestamp = datetime.now().strftime("%d-%b-%Y-%H:%M:%S")
     wrong_records_path = os.path.join(
-        os.getcwd(),
-        f'{save_errors}_errors_{timestamp}.json'
+        os.getcwd(), f'{save_errors}_errors_{timestamp}.json'
     )
 
     with open(wrong_records_path, 'w') as _json:
@@ -165,13 +184,18 @@ def check_and_update_data_with_schema(data, ana):
     if not schema and not ana:
         click.secho(
             'You need to provide the --ana/-a parameter OR '
-            'add the $schema field in your JSON', fg='red')
+            'add the $schema field in your JSON',
+            fg='red',
+        )
         return False
 
     try:
         if schema:
             if ana:
-                click.secho("Your data already provide a $schema, --ana will not be used.")  # noqa
+                click.secho(
+                    "Your data already provide a $schema,"
+                    " --ana will not be used."
+                )
             resolve_schema_by_url(schema)
         elif ana:
             data['$schema'] = schema_name_to_url(ana)
@@ -201,12 +225,14 @@ def create_deposit_with_permissions(data, roles, users, owner, ana, errors):
                 for role in roles:
                     _role = get_existing_or_register_role(role.strip())
                     deposit._add_egroup_permissions(
-                        _role, ['deposit-read'], db.session)
+                        _role, ['deposit-read'], db.session
+                    )
             if users:
                 for user in users:
                     _user = get_existing_or_register_user(user.strip())
                     deposit._add_user_permissions(
-                        _user, ['deposit-read'], db.session)
+                        _user, ['deposit-read'], db.session
+                    )
 
             deposit.commit()
         except ValidationError as err:
@@ -215,4 +241,6 @@ def create_deposit_with_permissions(data, roles, users, owner, ana, errors):
             return
 
     db.session.commit()
-    click.secho(f"Created deposit with id: {deposit['_deposit']['id']}", fg='green')  # noqa
+    click.secho(
+        f"Created deposit with id: {deposit['_deposit']['id']}", fg='green'
+    )  # noqa

cap/modules/deposit/ext.py

@@ -1,10 +1,13 @@
 """Initialize extension."""
 
 from __future__ import absolute_import, print_function
+
 import json
 
 from invenio_files_rest.views import blueprint as files_blueprint
 from invenio_indexer.signals import before_record_index
+
+from cap.modules.deposit.cli import add, create_deposit
 from cap.modules.deposit.utils import fix_bucket_links, prepare_record
 
 
@@ -24,11 +27,13 @@ def update_file_links(response):
             try:
                 if response.content_type == 'application/json':
                     resp_json = json.loads(response.data)
-                    response.data = json.dumps(
-                        fix_bucket_links(resp_json))
+                    response.data = json.dumps(fix_bucket_links(resp_json))
             finally:
                 return response
 
         before_record_index.connect(prepare_record, sender=app)
 
+        app.cli.add_command(add)
+        app.cli.add_command(create_deposit)
+
         app.extensions['cap_deposit'] = self

cap/modules/deposit/links.py

@@ -30,11 +30,10 @@
 from flask import current_app, request
 from invenio_records_files.links import default_bucket_link_factory
 
+from cap.modules.deposit.api import CAPDeposit
+from cap.modules.deposit.utils import extract_actions_from_class
 from cap.modules.records.utils import api_url_for, url_to_api_url
 
-from .api import CAPDeposit
-from .utils import extract_actions_from_class
-
 
 @cached(LRUCache(maxsize=1024), key=lambda pid, **kwargs: hashkey(str(pid)))
 def links_factory(pid, record=None, record_hit=None, **kwargs):
@@ -46,7 +45,7 @@ def links_factory(pid, record=None, record_hit=None, **kwargs):
             host=request.host,
             scheme=request.scheme,
             pid_value=pid.pid_value,
-        )
+        ),
     }
 
     try:
@@ -55,7 +54,8 @@ def links_factory(pid, record=None, record_hit=None, **kwargs):
             links['bucket'] = url_to_api_url(bucket_link)
     except Exception:
         current_app.logger.info(
-            f'Bucket link generation error for deposit: {pid.pid_value}')
+            f'Bucket link generation error for deposit: {pid.pid_value}'
+        )
 
     for action in extract_actions_from_class(CAPDeposit):
         if action != "review":

cap/modules/deposit/minters.py

@@ -30,13 +30,15 @@
 
 from invenio_pidstore.models import PersistentIdentifier, PIDStatus
 
+from cap.modules.deposit.utils import (
+    generate_auto_incremental_pid,
+    set_copy_to_attr,
+)
 from cap.modules.schemas.resolvers import (
     resolve_schema_by_name_and_version,
     resolve_schema_by_url,
 )
 
-from .utils import generate_auto_incremental_pid, set_copy_to_attr
-
 
 def cap_deposit_minter(record_uuid, data, schema=None):
     """Mint deposit's identifier."""

cap/modules/deposit/permissions.py

@@ -33,6 +33,7 @@
 from invenio_records_files.models import RecordsBuckets
 from sqlalchemy.orm.exc import NoResultFound
 
+from cap.modules.deposit.errors import WrongJSONSchemaError
 from cap.modules.experiments.permissions import cms_pag_convener_action
 from cap.modules.records.permissions import RecordFilesPermission
 from cap.modules.schemas.models import Schema
@@ -48,8 +49,6 @@
 )
 from cap.modules.schemas.resolvers import resolve_schema_by_url
 
-from .errors import WrongJSONSchemaError
-
 DepositReadActionNeed = partial(ParameterizedActionNeed, 'deposit-read')
 """Action need for reading a record."""
 

cap/modules/deposit/resolvers.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of CERN Analysis Preservation Framework.
+# Copyright (C) 2016, 2017 CERN.
+#
+# CERN Analysis Preservation Framework is free software; you can redistribute
+# it and/or modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# CERN Analysis Preservation Framework is distributed in the hope that it will
+# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with CERN Analysis Preservation Framework; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+# MA 02111-1307, USA.
+#
+# In applying this license, CERN does not
+# waive the privileges and immunities granted to it by virtue of its status
+# as an Intergovernmental Organization or submit itself to any jurisdiction.
+"""Resolver for JSON Schemas."""
+
+from __future__ import absolute_import, print_function
+
+import jsonresolver
+from invenio_pidstore.resolver import Resolver
+
+# from cap.modules.deposit.api import CAPDeposit
+# from cap.config import _PID
+
+
+# @jsonresolver.route(f'/api/deposits/<{_PID}:pid_value>',
+@jsonresolver.route(
+    '/api/deposits/<pid_value>', host='analysispreservation.cern.ch'
+)
+def resolve_api(pid_value):
+    """Resolve CAP JSON schemas."""
+    return resolve_by_deposit_id(pid_value)
+
+
+def resolve_by_deposit_id(depid):
+    from cap.modules.deposit.api import CAPDeposit
+    from cap.modules.records.serializers import record_metadata_json_v1
+
+    resolver = Resolver(pid_type='depid', object_type='rec', getter=lambda x: x)
+
+    dep, rec_uuid = resolver.resolve(depid)
+    rec = CAPDeposit.get_record(rec_uuid)
+
+    data = record_metadata_json_v1.transform_record(dep, rec)
+    return data