Merge branch 'master' of github.com:materialsproject/maggma into mult…

…i_key

requirements-docs.txt

@@ -1,5 +1,5 @@
 mkdocs==1.1.2
-mkdocs-material==5.3.0
+mkdocs-material==5.3.2
 mkdocs-material-components==1.10.14
 mkdocs-minify-plugin==0.3.0
-mkdocstrings==0.11.4
+mkdocstrings==0.12.0

requirements-optional.txt

@@ -1,3 +1,3 @@
 uvicorn==0.11.5
-boto3==1.14.2
-hvac==0.10.3
+boto3==1.14.7
+hvac==0.10.4

requirements-testing.txt

@@ -7,5 +7,5 @@ moto==1.3.14
 pycodestyle==2.6.0
 pydocstyle==5.0.2
 flake8==3.8.3
-mypy==0.780
+mypy==0.781
 mypy-extensions==0.4.3

requirements.txt

@@ -8,7 +8,7 @@ mongogrant==0.3.1
 aioitertools==0.7.0
 pydantic==1.5.1
 fastapi==0.58.0
-numpy==1.18.5
+numpy==1.19.0
 pynng==0.5.0
 dnspython==1.16.0
 uvicorn==0.11.5

setup.py

@@ -43,7 +43,7 @@
         "sshtunnel>=0.1.5",
         "msgpack>=0.5.6",
     ],
-    extras_require={"vault": ["hvac>=0.9.5"], "S3": ["boto3==1.14.2"]},
+    extras_require={"vault": ["hvac>=0.9.5"], "S3": ["boto3==1.14.7"]},
     classifiers=[
         "Programming Language :: Python :: 3",
         "Programming Language :: Python :: 3.6",

src/maggma/stores/aws.py

@@ -9,13 +9,12 @@
 from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
 
 import msgpack  # type: ignore
+from maggma.core import Sort, Store
+from maggma.utils import grouper, to_isoformat_ceil_ms
 from monty.dev import deprecated
 from monty.msgpack import default as monty_default
 from monty.msgpack import object_hook as monty_object_hook
 
-from maggma.core import Sort, Store
-from maggma.utils import grouper, to_isoformat_ceil_ms
-
 try:
     import botocore
     import boto3
@@ -361,6 +360,30 @@ def rebuild_index_from_s3_data(self):
 
         self.index.update(index_docs)
 
+    def rebuild_metadata_from_index(self, index_query: dict = None):
+        """
+        Read data from the index store and populate the metadata of the S3 bucket
+        Force all of the keys to be lower case to be Minio compatible
+        Args:
+            index_query: query on the index store
+        """
+
+        qq = {} if index_query is None else index_query
+        for index_doc in self.index.query(qq):
+            key_ = self.sub_dir + index_doc[self.key]
+            s3_object = self.s3_bucket.Object(key_)
+            # make sure the keys all all lower case
+            new_meta = {str(k).lower(): v for k, v in s3_object.metadata.items()}
+            for k, v in index_doc.items():
+                new_meta[str(k).lower()] = v
+            new_meta.pop("_id")
+            # s3_object.metadata.update(new_meta)
+            s3_object.copy_from(
+                CopySource={"Bucket": self.s3_bucket.name, "Key": key_},
+                Metadata=new_meta,
+                MetadataDirective="REPLACE",
+            )
+
     def __eq__(self, other: object) -> bool:
         """
         Check equality for S3Store

tests/stores/test_aws.py

@@ -2,12 +2,12 @@
 
 import boto3
 import msgpack
-import pytest
 from botocore.exceptions import ClientError
+from maggma.stores import MemoryStore, MongoStore, S3Store
 from monty.msgpack import default, object_hook
 from moto import mock_s3
 
-from maggma.stores import MemoryStore, MongoStore, S3Store
+import pytest
 
 
 @pytest.fixture
@@ -80,6 +80,14 @@ def test_update(s3store):
     assert s3store.query_one({"task_id": "mp-4"})["data"] == "asd"
 
 
+def test_rebuild_meta_from_index(s3store):
+    s3store.update([{"task_id": "mp-2", "data": "asd"}])
+    s3store.index.update({"task_id": "mp-2", "add_meta": "hello"})
+    s3store.rebuild_metadata_from_index()
+    s3_object = s3store.s3_bucket.Object("mp-2")
+    assert s3_object.metadata["add_meta"] == "hello"
+
+
 def test_remove(s3store):
     s3store.update([{"task_id": "mp-2", "data": "asd"}])
     s3store.update([{"task_id": "mp-4", "data": "asd"}])