Merge pull request #145 from multinet-app/csv-delimiters

Add a delimiter field to the csv upload for user specification

Author: JackWilb

multinet/api/tasks/upload/csv.py

@@ -38,14 +38,25 @@ def process_row(row: Dict[str, Any], cols: Dict[str, TableTypeAnnotation.Type])
 
 @shared_task(base=ProcessUploadTask)
 def process_csv(
-    task_id: int, table_name: str, edge: bool, columns: Dict[str, TableTypeAnnotation.Type]
+    task_id: int,
+    table_name: str,
+    edge: bool,
+    columns: Dict[str, TableTypeAnnotation.Type],
+    delimiter: str,
+    quotechar: str,
 ) -> None:
     upload: Upload = Upload.objects.get(id=task_id)
 
     # Download data from S3/MinIO
     with upload.blob as blob_file:
         blob_file: BinaryIO = blob_file
-        csv_rows = list(csv.DictReader(StringIO(blob_file.read().decode('utf-8'))))
+        csv_rows = list(
+            csv.DictReader(
+                StringIO(blob_file.read().decode('utf-8')),
+                delimiter=delimiter,
+                quotechar=quotechar,
+            )
+        )
 
     # Cast entries in each row to appropriate type, if necessary
     for i, row in enumerate(csv_rows):

multinet/api/tests/test_upload_csv.py

@@ -61,6 +61,8 @@ def airports_csv(
                 'timezone': 'number',
                 'year built': 'number',
             },
+            'delimiter': ',',
+            'quotechar': '\"',
         },
         format='json',
     )
@@ -106,6 +108,8 @@ def test_create_upload_model_invalid_columns(
             'edge': False,
             'table_name': 'table',
             'columns': {'foo': 'invalid'},
+            'delimiter': ',',
+            'quotechar': '\"',
         },
         format='json',
     )
@@ -114,6 +118,64 @@ def test_create_upload_model_invalid_columns(
     assert r.json() == {'columns': {'foo': ['"invalid" is not a valid choice.']}}
 
 
+@pytest.mark.django_db
+def test_create_upload_model_missing_delimiter(
+    workspace: Workspace, user: User, authenticated_api_client
+):
+    workspace.set_user_permission(user, WorkspaceRoleChoice.WRITER)
+    r: Response = authenticated_api_client.post(
+        f'/api/workspaces/{workspace.name}/uploads/csv/',
+        {
+            # Not an issue to specify invalid field_value, as that's checked after columns,
+            # so the request will return before that is checked
+            'field_value': 'field_value',
+            'edge': False,
+            'table_name': 'table',
+            'columns': {
+                'latitude': 'number',
+                'longitude': 'number',
+                'altitude': 'number',
+                'timezone': 'number',
+                'year built': 'number',
+            },
+            'quotechar': '\"',
+        },
+        format='json',
+    )
+
+    assert r.status_code == 400
+    assert r.json() == {'delimiter': ['This field is required.']}
+
+
+@pytest.mark.django_db
+def test_create_upload_model_missing_quotechar(
+    workspace: Workspace, user: User, authenticated_api_client
+):
+    workspace.set_user_permission(user, WorkspaceRoleChoice.WRITER)
+    r: Response = authenticated_api_client.post(
+        f'/api/workspaces/{workspace.name}/uploads/csv/',
+        {
+            # Not an issue to specify invalid field_value, as that's checked after columns,
+            # so the request will return before that is checked
+            'field_value': 'field_value',
+            'edge': False,
+            'table_name': 'table',
+            'columns': {
+                'latitude': 'number',
+                'longitude': 'number',
+                'altitude': 'number',
+                'timezone': 'number',
+                'year built': 'number',
+            },
+            'delimiter': ',',
+        },
+        format='json',
+    )
+
+    assert r.status_code == 400
+    assert r.json() == {'quotechar': ['This field is required.']}
+
+
 @pytest.mark.django_db
 @pytest.mark.parametrize('permission,status_code', [(None, 404), (WorkspaceRoleChoice.READER, 403)])
 def test_create_upload_model_csv_invalid_permissions(
@@ -164,6 +226,8 @@ def test_create_upload_model_invalid_field_value(
             'field_value': 'field_value',
             'edge': False,
             'table_name': 'table',
+            'delimiter': ',',
+            'quotechar': '\"',
         },
         format='json',
     )

multinet/api/views/serializers.py

@@ -255,6 +255,8 @@ class CSVUploadCreateSerializer(UploadCreateSerializer):
         child=serializers.ChoiceField(choices=TableTypeAnnotation.Type.choices),
         default=dict,
     )
+    delimiter = serializers.CharField()
+    quotechar = serializers.CharField()
 
 
 class D3JSONUploadCreateSerializer(UploadCreateSerializer):

multinet/api/views/upload.py

@@ -90,6 +90,8 @@ def upload_csv(self, request, parent_lookup_workspace__name: str):
             table_name=table_name,
             edge=serializer.validated_data['edge'],
             columns=serializer.validated_data['columns'],
+            delimiter=serializer.validated_data['delimiter'],
+            quotechar=serializer.validated_data['quotechar'],
         )
 
         return Response(UploadReturnSerializer(upload).data, status=status.HTTP_200_OK)