Project sync (#413)

* Implemented project/sync endpoint

* Added record.dataLastUpdated

* Removed unused isDataChanged

* Ensure that file exists for indexing earlier

* Implemented get_file_last_updated

* Use/set update_at during indexing

* Updated the client

* Implemented project/sync test

* Clarify variable naming

* Fixed a test name

client/client.ts

@@ -250,7 +250,7 @@ export class Client {
 
   async projectSync(props: Record<string, never>) {
     const result = await this.request('/project/sync', props)
-    return result as Record<string, never>
+    return result as { files: types.IFile[] }
   }
 
   // Resource

client/components/Application/store.ts

@@ -6,6 +6,7 @@ import { assert } from 'ts-essentials'
 import { Client } from '../../client'
 import { ApplicationProps } from './index'
 import { IDialog } from './types'
+import * as settings from '../../settings'
 import * as helpers from '../../helpers'
 import * as types from '../../types'
 
@@ -83,9 +84,11 @@ export function makeStore(props: ApplicationProps) {
 
     onStart: async () => {
       const { client, loadConfig, loadFiles, updateState } = get()
+      updateState({ dialog: 'start' })
+
+      // Wait for the server
       // @ts-ignore
       const sendFatalError = window?.opendataeditor?.sendFatalError
-      updateState({ dialog: 'start' })
       let ready = false
       let attempt = 0
       const maxAttempts = sendFatalError ? 300 : 3
@@ -105,6 +108,13 @@ export function makeStore(props: ApplicationProps) {
           await delay(delaySeconds * 1000)
         }
       }
+
+      // Setup project sync polling
+      setInterval(async () => {
+        const { files } = await client.projectSync({})
+        updateState({ files })
+      }, settings.PROJECT_SYNC_INTERVAL_MILLIS)
+
       updateState({ dialog: undefined })
     },
     onFileCreate: async (paths) => {

client/settings.ts

@@ -31,6 +31,7 @@ export const FALSE_VALUES = ['false', 'FALSE', 'no', 'NO', '0']
 export const METADATA_FORMATS = ['yaml', 'json']
 export const METADATA_TYPES = ['resource', 'dialect', 'schema', 'checklist', 'pipeline']
 export const MAX_TABLE_SOURCE_SIZE = 1000000
+export const PROJECT_SYNC_INTERVAL_MILLIS = 10 * 1000
 export const FILE_TYPES = [
   'article',
   'chart',

client/types/record.ts

@@ -4,5 +4,6 @@ export interface IRecord {
   name: string
   type: string
   path: string
+  dataUpdatedAt?: number
   resource: IResource
 }

server/endpoints/file/index.py

@@ -29,6 +29,12 @@ def action(project: Project, props: Props) -> Result:
     md = project.metadata
     db = project.database
 
+    # Ensure file exists
+    fullpath = fs.get_fullpath(props.path)
+    if not fullpath.is_file():
+        raise FrictionlessException("file not found")
+    data_updated_at = helpers.get_file_updated_at(project, path=props.path)
+
     # Read current state
     record = helpers.read_record(project, path=props.path)
     report = db.read_artifact(name=record.name, type="report") if record else None
@@ -40,15 +46,17 @@ def action(project: Project, props: Props) -> Result:
     missing_report = not report
     missing_measure = not measure
     missing_table = record and record.type == "table" and table is None
+    is_data_outdated = record and (record.dataUpdatedAt or 0) < data_updated_at
 
     # Ensure indexing
-    if missing_record or missing_report or missing_measure or missing_table:
-        # Ensure file exists
-        fullpath = fs.get_fullpath(props.path)
-        if not fullpath.is_file():
-            raise FrictionlessException("file not found")
-
-        # Index resource
+    if (
+        missing_record
+        or missing_report
+        or missing_measure
+        or missing_table
+        or is_data_outdated
+    ):
+        # Create resource
         path, basepath = fs.get_path_and_basepath(props.path)
         name = helpers.name_record(project, path=path)
         resource_obj = (
@@ -61,7 +69,7 @@ def action(project: Project, props: Props) -> Result:
             )
         )
 
-        # Validate resource
+        # Index/validate resource
         report_obj = helpers.index_resource(
             project, resource=resource_obj, table_name=name
         )
@@ -76,11 +84,10 @@ def action(project: Project, props: Props) -> Result:
                 resource=resource_obj.to_descriptor(),
             )
         record.resource = resource_obj.to_descriptor()
+        record.dataUpdatedAt = data_updated_at
 
         # Create measure
-        measure_obj = models.Measure(
-            errors=report_obj.stats["errors"],
-        )
+        measure_obj = models.Measure(errors=report_obj.stats["errors"])
         measure = measure_obj.model_dump()
 
         # Write document/artifacts

server/endpoints/json/patch.py

@@ -38,7 +38,6 @@ def action(project: Project, props: Props) -> Result:
         path=props.path,
         toPath=props.toPath,
         resource=props.resource,
-        isDataChanged=props.data is not None,
     )
 
     # Write contents

server/endpoints/project/__spec__/test_sync.py

@@ -0,0 +1,33 @@
+from pathlib import Path
+
+import pytest
+
+from server import helpers, models
+from server.fixtures import bytes1, folder1, name1, name2, not_secure
+
+# Action
+
+
+def test_server_project_sync(client):
+    # Create two files
+    path1 = client("/file/create", path=name1, bytes=bytes1).path
+    path2 = client("/file/create", path=name2, bytes=bytes1).path
+    assert client("/file/list").files == [
+        models.File(path=name1, type="file"),
+        models.File(path=name2, type="file"),
+    ]
+
+    # Index files
+    client("/file/index", path=name1)
+    client("/file/index", path=name2)
+
+    # Delete one file not using the API and sync the project
+    (client.project.public / name2).unlink()
+    client("/project/sync")
+    assert client("/file/list").files == [
+        models.File(path=name1, type="text", name="name1", errors=0),
+    ]
+
+    # It should have deleted the record of the deleted file
+    assert len(list(client.project.metadata.iter_documents(type="record"))) == 1
+    assert client.project.metadata.read_document(name="name2", type="record") is None

server/endpoints/project/sync.py

@@ -1 +1,43 @@
-# TODO: implement
+from __future__ import annotations
+
+from typing import List, Optional
+
+from fastapi import Request
+from pydantic import BaseModel
+
+from ... import helpers, models
+from ...project import Project
+from ...router import router
+
+
+class Props(BaseModel, extra="forbid"):
+    pass
+
+
+class Result(BaseModel, extra="forbid"):
+    files: List[models.File]
+
+
+@router.post("/project/sync")
+def endpoint(request: Request, props: Props) -> Result:
+    return action(request.app.get_project())
+
+
+def action(project: Project, props: Optional[Props] = None) -> Result:
+    md = project.metadata
+    from ... import endpoints
+
+    # Get all the project's file paths that actually exist on the disc at this moment
+    paths: List[str] = []
+    result = endpoints.file.list.action(project)
+    for file in result.files:
+        paths.append(file.path)
+
+    # Delete all the records that are not in the list of the paths
+    # e.g. they were deleted by a user outside of the app
+    for descriptor in md.iter_documents(type="record"):
+        path = descriptor["path"]
+        if path not in paths:
+            helpers.delete_record(project, path=path)
+
+    return Result(files=result.files)

server/endpoints/table/patch.py

@@ -44,7 +44,6 @@ def action(project: Project, props: Props) -> Result:
         path=props.path,
         toPath=props.toPath,
         resource=props.resource,
-        isDataChanged=props.history is not None,
     )
 
     # Copy table

server/endpoints/text/patch.py

@@ -38,7 +38,6 @@ def action(project: Project, props: Props) -> Result:
         path=props.path,
         toPath=props.toPath,
         resource=props.resource,
-        isDataChanged=props.text is not None,
     )
 
     # Write contents

server/helpers/file.py

@@ -48,6 +48,12 @@ def write_file(
     return path
 
 
+def get_file_updated_at(project: Project, *, path: str):
+    fs = project.filesystem
+    fullpath = fs.get_fullpath(path)
+    return fullpath.stat().st_mtime
+
+
 def create_file_filter(project: Project) -> Callable[[str], bool]:
     fs = project.filesystem
     fullpath = fs.get_fullpath(".gitignore")

server/helpers/record.py

@@ -23,7 +23,6 @@ def patch_record(
     type: Optional[str] = None,
     resource: Optional[types.IDescriptor] = None,
     toPath: Optional[str] = None,
-    isDataChanged: bool = False,
 ):
     md = project.metadata
 

server/models/history.py

@@ -19,11 +19,13 @@ class CellUpdate(Change):
     fieldName: str
     value: Any
 
+
 class Cell(BaseModel):
     rowNumber: int
     fieldName: str
     value: Any
 
+
 class MultipleCellUpdate(Change):
     type: Literal["multiple-cells-update"]
     cells: List[Cell]
@@ -32,7 +34,6 @@ class MultipleCellUpdate(Change):
 class History(BaseModel):
     changes: List[
         Annotated[
-            Union[RowDelete, CellUpdate, MultipleCellUpdate],
-            Field(discriminator="type")
+            Union[RowDelete, CellUpdate, MultipleCellUpdate], Field(discriminator="type")
         ]
     ]

server/models/record.py

@@ -1,10 +1,34 @@
+from typing import Optional
+
 from pydantic import BaseModel
 
 from .. import types
 
 
 class Record(BaseModel):
     name: str
+    """
+    Unique file name/identifier across the project.
+    """
+
     type: str
+    """
+    File type e.g. "text" or "table".
+    """
+
     path: str
+    """
+    Path to the file relative to the project root.
+    """
+
+    dataUpdatedAt: Optional[float] = None
+    """
+    UNIX timestamp of the last file update.
+    It's used to detect changes that were made outside of the app.
+    """
+
     resource: types.IDescriptor
+    """
+    Data Resource descriptor as per the Data Package Standard:
+    https://datapackage.org/specifications/data-resource/
+    """