Merge pull request #1216 from schemacrawler/chatgpt

ChatGPT enhancements

.github/workflows/publish-website.yml

@@ -67,7 +67,7 @@ jobs:
 
     - id: upload-artifact
       name: Upload artifact
-      uses: actions/upload-pages-artifact@v1
+      uses: actions/upload-pages-artifact@v2
       with:
         path: ./schemacrawler-website/target/_website
 

schemacrawler-api/src/main/java/schemacrawler/crawl/MutableCatalog.java

@@ -299,18 +299,49 @@ public <N extends NamedObject> void reduce(final Class<N> clazz, final Reducer<N
     requireNonNull(clazz, "No lookup class provided");
 
     if (Schema.class.isAssignableFrom(clazz)) {
-      ((Reducer<Schema>) reducer).reduce(schemas);
+      final Reducer<Schema> schemaReducer = (Reducer<Schema>) reducer;
+      schemaReducer.reduce(schemas);
     } else if (Table.class.isAssignableFrom(clazz)) {
       // Filter the list of tables based on grep criteria, and
       // parent-child relationships
-      ((Reducer<Table>) reducer).reduce(tables);
+      final Reducer<Table> tableReducer = (Reducer<Table>) reducer;
+      tableReducer.reduce(tables);
     } else if (Routine.class.isAssignableFrom(clazz)) {
       // Filter the list of routines based on grep criteria
-      ((Reducer<Routine>) reducer).reduce(routines);
+      final Reducer<Routine> routineReducer = (Reducer<Routine>) reducer;
+      routineReducer.reduce(routines);
     } else if (Synonym.class.isAssignableFrom(clazz)) {
-      ((Reducer<Synonym>) reducer).reduce(synonyms);
+      final Reducer<Synonym> synonymReducer = (Reducer<Synonym>) reducer;
+      synonymReducer.reduce(synonyms);
     } else if (Sequence.class.isAssignableFrom(clazz)) {
-      ((Reducer<Sequence>) reducer).reduce(sequences);
+      final Reducer<Sequence> sequenceReducer = (Reducer<Sequence>) reducer;
+      sequenceReducer.reduce(sequences);
+    }
+  }
+
+  @Override
+  public <N extends NamedObject> void undo(final Class<N> clazz, final Reducer<N> reducer) {
+    requireNonNull(reducer, "No reducer provided");
+    requireNonNull(clazz, "No lookup class provided");
+
+    if (Schema.class.isAssignableFrom(clazz)) {
+      final Reducer<Schema> schemaReducer = (Reducer<Schema>) reducer;
+      schemaReducer.undo(schemas);
+    } else if (Table.class.isAssignableFrom(clazz)) {
+      // Filter the list of tables based on grep criteria, and
+      // parent-child relationships
+      final Reducer<Table> tableReducer = (Reducer<Table>) reducer;
+      tableReducer.undo(tables);
+    } else if (Routine.class.isAssignableFrom(clazz)) {
+      // Filter the list of routines based on grep criteria
+      final Reducer<Routine> routineReducer = (Reducer<Routine>) reducer;
+      routineReducer.undo(routines);
+    } else if (Synonym.class.isAssignableFrom(clazz)) {
+      final Reducer<Synonym> synonymReducer = (Reducer<Synonym>) reducer;
+      synonymReducer.undo(synonyms);
+    } else if (Sequence.class.isAssignableFrom(clazz)) {
+      final Reducer<Sequence> sequenceReducer = (Reducer<Sequence>) reducer;
+      sequenceReducer.undo(sequences);
     }
   }
 

schemacrawler-api/src/main/java/schemacrawler/crawl/NamedObjectList.java

@@ -56,6 +56,8 @@ final class NamedObjectList<N extends NamedObject> implements Serializable, Redu
 
   private static final long serialVersionUID = 3257847666804142128L;
 
+  private static final String SCHEMACRAWLER_FILTERED_OUT = "schemacrawler.filtered_out";
+
   private static NamedObjectKey makeLookupKey(final NamedObject namedObject) {
     final NamedObjectKey key;
     if (namedObject == null) {
@@ -75,9 +77,11 @@ private static NamedObjectKey makeLookupKey(final NamedObject namedObject, final
   }
 
   private final Map<NamedObjectKey, N> objects = new ConcurrentHashMap<>();
+  private final Map<NamedObjectKey, N> filteredObjects = new ConcurrentHashMap<>();
 
+  /** {@inheritDoc} */
   @Override
-  public void filter(final Predicate<? super N> predicate) {
+  public synchronized void filter(final Predicate<? super N> predicate) {
     if (predicate == null) {
       return;
     }
@@ -86,12 +90,15 @@ public void filter(final Predicate<? super N> predicate) {
     for (final Iterator<Entry<NamedObjectKey, N>> iterator = entrySet.iterator();
         iterator.hasNext(); ) {
       final Entry<NamedObjectKey, N> entry = iterator.next();
+      final NamedObjectKey namedObjectKey = entry.getKey();
       final N namedObject = entry.getValue();
       if (!predicate.test(namedObject)) {
+        // Filter object by moving it to the filtered objects map
         iterator.remove();
+        filteredObjects.put(namedObjectKey, namedObject);
         if (namedObject instanceof AttributedObject) {
           final AttributedObject attributedObject = (AttributedObject) namedObject;
-          attributedObject.setAttribute("schemacrawler.filtered_out", true);
+          attributedObject.setAttribute(SCHEMACRAWLER_FILTERED_OUT, true);
         }
       }
     }
@@ -127,6 +134,24 @@ public void remove() {
     return new UnmodifiableIterator(values().iterator());
   }
 
+  /** {@inheritDoc} */
+  @Override
+  public synchronized void resetFilter() {
+    final Set<Entry<NamedObjectKey, N>> entrySet = filteredObjects.entrySet();
+    for (final Iterator<Entry<NamedObjectKey, N>> iterator = entrySet.iterator();
+        iterator.hasNext(); ) {
+      final Entry<NamedObjectKey, N> entry = iterator.next();
+      final NamedObjectKey namedObjectKey = entry.getKey();
+      final N namedObject = entry.getValue();
+      objects.put(namedObjectKey, namedObject);
+      iterator.remove();
+      if (namedObject instanceof AttributedObject) {
+        final AttributedObject attributedObject = (AttributedObject) namedObject;
+        attributedObject.removeAttribute(SCHEMACRAWLER_FILTERED_OUT);
+      }
+    }
+  }
+
   /** {@inheritDoc} */
   @Override
   public String toString() {

schemacrawler-api/src/main/java/schemacrawler/filter/ReducerFactory.java

@@ -6,9 +6,7 @@
 import static schemacrawler.filter.FilterFactory.sequenceFilter;
 import static schemacrawler.filter.FilterFactory.synonymFilter;
 import static schemacrawler.filter.FilterFactory.tableFilter;
-
 import java.util.function.Predicate;
-
 import schemacrawler.schema.NamedObject;
 import schemacrawler.schema.Reducer;
 import schemacrawler.schema.ReducibleCollection;
@@ -34,6 +32,12 @@ public void reduce(final ReducibleCollection<? extends N> allNamedObjects) {
       requireNonNull(allNamedObjects, "No named objects provided");
       allNamedObjects.filter(filter);
     }
+
+    @Override
+    public void undo(final ReducibleCollection<? extends N> allNamedObjects) {
+      requireNonNull(allNamedObjects, "No named objects provided");
+      allNamedObjects.resetFilter();
+    }
   }
 
   public static Reducer<Routine> getRoutineReducer(final SchemaCrawlerOptions options) {

schemacrawler-api/src/main/java/schemacrawler/filter/TablesReducer.java

@@ -29,12 +29,10 @@
 package schemacrawler.filter;
 
 import static java.util.Objects.requireNonNull;
-
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.function.Predicate;
-
 import schemacrawler.schema.PartialDatabaseObject;
 import schemacrawler.schema.Reducer;
 import schemacrawler.schema.ReducibleCollection;
@@ -61,6 +59,14 @@ public void reduce(final ReducibleCollection<? extends Table> allTables) {
     doReduce(allTables);
   }
 
+  @Override
+  public void undo(final ReducibleCollection<? extends Table> allTables) {
+    if (allTables == null) {
+      return;
+    }
+    allTables.resetFilter();
+  }
+
   private void doReduce(final ReducibleCollection<? extends Table> allTables) {
     // Filter tables, keeping the ones we need
     final Set<Table> reducedTables = new HashSet<>();

schemacrawler-api/src/main/java/schemacrawler/schema/Reducer.java

@@ -28,8 +28,9 @@
 
 package schemacrawler.schema;
 
-@FunctionalInterface
 public interface Reducer<N extends NamedObject> {
 
   void reduce(final ReducibleCollection<? extends N> namedObjects);
+
+  void undo(final ReducibleCollection<? extends N> namedObjects);
 }

schemacrawler-api/src/main/java/schemacrawler/schema/Reducible.java

@@ -28,8 +28,9 @@
 
 package schemacrawler.schema;
 
-@FunctionalInterface
 public interface Reducible {
 
   <N extends NamedObject> void reduce(Class<N> clazz, Reducer<N> reducer);
+
+  <N extends NamedObject> void undo(Class<N> clazz, Reducer<N> reducer);
 }

schemacrawler-api/src/main/java/schemacrawler/schema/ReducibleCollection.java

@@ -32,5 +32,13 @@
 
 public interface ReducibleCollection<N extends NamedObject> extends Iterable<N> {
 
+  /**
+   * Filter out objects.
+   *
+   * @param predicate Predicate to filter by.
+   */
   void filter(Predicate<? super N> predicate);
+
+  /** Reset all previously filtered objects. */
+  void resetFilter();
 }

schemacrawler-api/src/main/java/schemacrawler/utility/MetaDataUtility.java

@@ -29,24 +29,34 @@
 package schemacrawler.utility;
 
 import static java.util.Objects.requireNonNull;
-
+import static schemacrawler.filter.ReducerFactory.getRoutineReducer;
+import static schemacrawler.filter.ReducerFactory.getSchemaReducer;
+import static schemacrawler.filter.ReducerFactory.getSequenceReducer;
+import static schemacrawler.filter.ReducerFactory.getSynonymReducer;
+import static schemacrawler.filter.ReducerFactory.getTableReducer;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
-
+import schemacrawler.schema.Catalog;
 import schemacrawler.schema.Column;
 import schemacrawler.schema.ColumnReference;
 import schemacrawler.schema.Index;
 import schemacrawler.schema.IndexColumn;
 import schemacrawler.schema.JavaSqlTypeGroup;
 import schemacrawler.schema.PartialDatabaseObject;
+import schemacrawler.schema.Reducible;
+import schemacrawler.schema.Routine;
+import schemacrawler.schema.Schema;
+import schemacrawler.schema.Sequence;
+import schemacrawler.schema.Synonym;
 import schemacrawler.schema.Table;
 import schemacrawler.schema.TableConstraint;
 import schemacrawler.schema.TableConstraintColumn;
 import schemacrawler.schema.TableReference;
 import schemacrawler.schema.TableRelationshipType;
 import schemacrawler.schemacrawler.Identifiers;
+import schemacrawler.schemacrawler.SchemaCrawlerOptions;
 import us.fatehi.utility.UtilityMarker;
 
 @UtilityMarker
@@ -237,6 +247,27 @@ public static String joinColumns(
     return String.join(", ", columnsList);
   }
 
+  public static void reduceCatalog(
+      final Catalog catalog, final SchemaCrawlerOptions schemaCrawlerOptions) {
+    requireNonNull(catalog, "No catalog provided");
+    requireNonNull(schemaCrawlerOptions, "No SchemaCrawler options provided");
+
+    ((Reducible) catalog).undo(Schema.class, getSchemaReducer(schemaCrawlerOptions));
+    ((Reducible) catalog).reduce(Schema.class, getSchemaReducer(schemaCrawlerOptions));
+
+    ((Reducible) catalog).undo(Table.class, getTableReducer(schemaCrawlerOptions));
+    ((Reducible) catalog).reduce(Table.class, getTableReducer(schemaCrawlerOptions));
+
+    ((Reducible) catalog).undo(Routine.class, getRoutineReducer(schemaCrawlerOptions));
+    ((Reducible) catalog).reduce(Routine.class, getRoutineReducer(schemaCrawlerOptions));
+
+    ((Reducible) catalog).undo(Synonym.class, getSynonymReducer(schemaCrawlerOptions));
+    ((Reducible) catalog).reduce(Synonym.class, getSynonymReducer(schemaCrawlerOptions));
+
+    ((Reducible) catalog).undo(Sequence.class, getSequenceReducer(schemaCrawlerOptions));
+    ((Reducible) catalog).reduce(Sequence.class, getSequenceReducer(schemaCrawlerOptions));
+  }
+
   public static Collection<List<String>> uniqueIndexCoumnNames(final Table table) {
     return indexCoumnNames(table, true);
   }

schemacrawler-api/src/test/java/schemacrawler/test/MetadataUtilityTest.java

@@ -36,16 +36,13 @@
 import static schemacrawler.schemacrawler.IdentifierQuotingStrategy.quote_all;
 import static schemacrawler.test.utility.DatabaseTestUtility.getCatalog;
 import static schemacrawler.test.utility.DatabaseTestUtility.schemaCrawlerOptionsWithMaximumSchemaInfoLevel;
-
 import java.sql.Connection;
 import java.util.List;
 import java.util.stream.Collectors;
-
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInstance;
 import org.junit.jupiter.api.TestInstance.Lifecycle;
-
 import schemacrawler.schema.Catalog;
 import schemacrawler.schema.ColumnReference;
 import schemacrawler.schema.ForeignKey;
@@ -56,7 +53,9 @@
 import schemacrawler.schema.TableRelationshipType;
 import schemacrawler.schemacrawler.Identifiers;
 import schemacrawler.schemacrawler.IdentifiersBuilder;
+import schemacrawler.schemacrawler.LimitOptionsBuilder;
 import schemacrawler.schemacrawler.SchemaCrawlerOptions;
+import schemacrawler.schemacrawler.SchemaCrawlerOptionsBuilder;
 import schemacrawler.test.utility.WithTestDatabase;
 import schemacrawler.utility.MetaDataUtility;
 import schemacrawler.utility.MetaDataUtility.ForeignKeyCardinality;
@@ -181,6 +180,31 @@ public void loadCatalog(final Connection connection) {
     }
   }
 
+  @Test
+  public void reduceCatalog() throws Exception {
+
+    final LimitOptionsBuilder limitOptionsBuilder = LimitOptionsBuilder.builder();
+    limitOptionsBuilder.includeTables(tableName -> !tableName.matches(".*\\.BOOKS"));
+
+    final SchemaCrawlerOptions schemaCrawlerOptions =
+        SchemaCrawlerOptionsBuilder.newSchemaCrawlerOptions()
+            .withLimitOptions(limitOptionsBuilder.toOptions());
+
+    // Reduce catalog
+    MetaDataUtility.reduceCatalog(catalog, schemaCrawlerOptions);
+
+    final Schema schema = catalog.lookupSchema("PUBLIC.BOOKS").get();
+    assertThat("BOOKS Schema not found", schema, notNullValue());
+
+    assertThat("BOOKS Table not found", !catalog.lookupTable(schema, "BOOKS").isPresent());
+
+    // Undo reduce catalog
+    MetaDataUtility.reduceCatalog(catalog, SchemaCrawlerOptionsBuilder.newSchemaCrawlerOptions());
+
+    final Table table = catalog.lookupTable(schema, "BOOKS").get();
+    assertThat("BOOKS Table not found", table, notNullValue());
+  }
+
   @Test
   public void tableUtilities() throws Exception {
 

schemacrawler-chatgpt/src/main/java/schemacrawler/tools/command/chatgpt/ChatGPTCommand.java

@@ -56,6 +56,12 @@ public void checkAvailability() throws RuntimeException {
 
   @Override
   public void execute() {
-    new ChatGPTConsole(commandOptions, catalog).console();
+    new ChatGPTConsole(commandOptions, catalog, connection).console();
+  }
+
+  @Override
+  public boolean usesConnection() {
+    // Support commands that use connections
+    return true;
   }
 }

schemacrawler-chatgpt/src/main/java/schemacrawler/tools/command/chatgpt/ChatGPTConsole.java

@@ -1,6 +1,7 @@
 package schemacrawler.tools.command.chatgpt;
 
 import static java.util.Objects.requireNonNull;
+import java.sql.Connection;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Scanner;
@@ -31,12 +32,15 @@ public final class ChatGPTConsole {
   private final OpenAiService service;
   private final CircularBoundedList<ChatMessage> chatHistory;
 
-  public ChatGPTConsole(final ChatGPTCommandOptions commandOptions, final Catalog catalog) {
+  public ChatGPTConsole(
+      final ChatGPTCommandOptions commandOptions,
+      final Catalog catalog,
+      final Connection connection) {
 
     this.commandOptions = requireNonNull(commandOptions, "ChatGPT options not provided");
     requireNonNull(catalog, "No catalog provided");
 
-    functionExecutor = ChatGPTUtility.newFunctionExecutor(catalog);
+    functionExecutor = ChatGPTUtility.newFunctionExecutor(catalog, connection);
     service = new OpenAiService(commandOptions.getApiKey());
     chatHistory = new CircularBoundedList<>(commandOptions.getContext());
   }