ci: add diffscrape package with Dart

.github/workflows/check-all-lint-rule-diffs.yaml

@@ -2,8 +2,6 @@ name: Check all lint rule diffs
 
 on:
   workflow_dispatch:
-  schedule:
-    - cron:  '0 0 15 * *'
 
 env:
   branch: "update-all-lint-rules"

.github/workflows/update-all-lint-rules.yaml

@@ -0,0 +1,73 @@
+name: Update all lint rules
+
+on:
+  schedule:
+    - cron:  '0 0 * * 3'
+  workflow_dispatch:
+
+env:
+  branch: "update-all-lint-rules"
+
+jobs:
+  update:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    outputs:
+      has_diff: ${{ steps.output.outputs.has_diff }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dart-lang/setup-dart@v1
+      - name: run
+        working-directory: ./packages/diffscrape
+        run: dart run diffscrape
+      - name: output diff
+        id: output
+        run: echo "diff_count=$(git diff --name-only --relative=packages/altive_lints/lib | wc -l)" >> "$GITHUB_ENV"
+      - name: Create branch
+        if: env.diff_count != '0'
+        run: |
+          git switch -c ${{ env.branch }}
+          git push origin ${{ env.branch }}
+      - name: Git config
+        if: env.diff_count != '0'
+        run: |
+          git remote set-url origin https://github-actions:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}
+          git config --global user.name "${GITHUB_ACTOR}"
+          git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com"
+      - name: Commit & Push
+        if: env.diff_count != '0'
+        run: |
+          git add .
+          git commit -m "feat: update all_lint_rules"
+          git push origin ${{ env.branch }}
+      - name: Output flag
+        run: echo "has_diff=${{ env.diff_count != '0' }}" >> "$GITHUB_OUTPUT"
+
+  pull-request:
+    name: Create Pull-Request
+    runs-on: ubuntu-latest
+    needs: [update]
+    if: ${{ needs.diff.outputs.has_diff == 'true' }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Generate GiHub App token
+        id: generate_token
+        uses: actions/create-github-app-token@v1
+        with:
+          app_id: ${{ secrets.PR_WRITER_APP_ID }}
+          private_key: ${{ secrets.PR_WRITER_PRIVATE_KEY }}
+
+      - name: Create PR
+        env:
+          GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }}
+        run: gh pr create -B $GITHUB_REF -t ${{ env.branch }} -b "" -a $GITHUB_ACTOR -H ${{ env.branch }}
+
+      - name: Revoke GitHub Apps token
+        env:
+          GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }}
+        run: |
+          curl --location --silent --request DELETE \
+            --url "${GITHUB_API_URL}/installation/token" \
+            --header "Accept: application/vnd.github+json" \
+            --header "X-GitHub-Api-Version: 2022-11-28" \
+            --header "Authorization: Bearer ${GITHUB_TOKEN}"

.vscode/launch.json

@@ -0,0 +1,24 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "diffscrape",
+            "cwd": "packages/diffscrape",
+            "request": "launch",
+            "type": "dart",
+            "args": [
+                "diffscrape",
+                "--uri",
+                "https://dart.dev/tools/linter-rules/all",
+                "--file",
+                "../altive_lints/lib/all_lint_rules.yaml",
+                "--query-selector",
+                "pre code.yaml",
+                "--verbose"
+            ]
+        }
+    ]
+}

.vscode/settings.json

@@ -8,7 +8,16 @@
         "gocolly",
         "interps",
         "ints",
+        "pubspec",
+        "redeclares",
         "tearoffs",
-        "todos"
-    ]
+        "todos",
+        "unawaited",
+        "writeln"
+    ],
+    "yaml.schemas": {
+        "https://json.schemastore.org/yamllint.json": [
+            "*.yaml",
+        ]
+    }
 }

packages/diffscrape/.gitignore

@@ -0,0 +1,3 @@
+# https://dart.dev/guides/libraries/private-files
+# Created by `dart pub`
+.dart_tool/

packages/diffscrape/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 0.0.1
+
+- Initial version.

packages/diffscrape/README.md

@@ -0,0 +1,26 @@
+# Diffscrape
+
+## What is it?
+
+Diffscrape is a tool for scraping web and file diffs.
+
+## Interface
+
+### uri
+URL of the scraping destination.
+
+### file
+Files to be compared. Any differences will be overwritten.
+
+### query-selector
+Selector for searching elements in HTML.
+
+A sample command-line application providing basic argument parsing with an entrypoint in `bin/`.
+
+```shell
+dart run diffscrape \
+--uri "https://dart.dev/tools/linter-rules/all" \
+--file "../altive_lints/lib/all_lint_rules.yaml" \
+--query-selector "pre code.yaml" \
+--verbose
+```

packages/diffscrape/analysis_options.yaml

@@ -0,0 +1,5 @@
+include: package:altive_lints/altive_lints.yaml
+
+linter:
+  rules:
+    - avoid_print: false

packages/diffscrape/bin/diffscrape.dart

@@ -0,0 +1,42 @@
+import 'package:args/args.dart';
+import 'package:diffscrape/diffscrape.dart';
+import 'package:diffscrape/src/arg.dart';
+import 'package:diffscrape/src/util.dart';
+
+const String version = '0.0.1';
+
+void main(List<String> arguments) {
+  final argParser = buildParser();
+  final ArgResults results;
+  try {
+    results = argParser.parse(arguments);
+  } on FormatException catch (e) {
+    // Print usage information if an invalid argument was provided.
+    print(e.message);
+    print('');
+    printUsage(argParser);
+    return;
+  }
+
+  if (results.wasParsed('help')) {
+    printUsage(argParser);
+    return;
+  }
+  if (results.wasParsed('version')) {
+    print('diffscrape version: $version');
+    return;
+  }
+
+  final verbose = results.wasParsed('verbose');
+
+  print('Positional arguments: ${results.rest}');
+  if (verbose) {
+    print('[VERBOSE] All arguments: ${results.arguments}');
+  }
+  scrape(
+    uriPath: results[argUri] as String,
+    filePath: results[argFile] as String,
+    querySelector: results[argQuerySelector] as String,
+    verbose: verbose,
+  );
+}

packages/diffscrape/lib/diffscrape.dart

@@ -0,0 +1,103 @@
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:html/parser.dart';
+import 'package:http/http.dart' as http;
+
+typedef Diff = Set<String>;
+typedef Diffs = ({Diff inserts, Diff deletes});
+const noDiff = (inserts: <String>{}, deletes: <String>{});
+
+/// [uriPath] のURLをスクレイピングして [filePath] のファイルと差分あれば、
+/// そのファイルをスクレイピングした内容で上書きする。
+///
+/// [uriPath] は、スクレイピング先のURLパス。
+/// [filePath] は、スクレイピング結果と比較し上書きしたいファイルのパス。
+/// [querySelector] は、スクレイピング先のHTMLのクエリセレクタ。
+/// [verbose] は、詳細な出力をするかどうか。
+Future<Diffs> scrape({
+  required String uriPath,
+  required String filePath,
+  required String querySelector,
+  required bool verbose,
+}) async {
+  final uri = Uri.tryParse(uriPath);
+  if (uri == null) {
+    stderr.writeln('Invalid URI: $uriPath');
+    exit(1);
+  }
+  if (!await FileSystemEntity.isFile(filePath)) {
+    stderr.writeln('Invalid file path: $filePath');
+    exit(1);
+  }
+  final file = File(filePath);
+
+  final oldSets = await _readFile(file, verbose);
+
+  final newRulesText = await _scrape(
+    uri: uri,
+    querySelector: querySelector,
+    verbose: verbose,
+  );
+  final newSets = newRulesText.split('\n').toSet();
+
+  if (oldSets == newSets) {
+    print('No diff.');
+    return noDiff;
+  }
+
+  final inserts = oldSets.difference(newSets);
+  print('inserts length: ${inserts.length}');
+  if (verbose && inserts.isNotEmpty) {
+    print('[VERBOSE] inserts: $inserts');
+  }
+  final deletes = newSets.difference(oldSets);
+  print('deletes length: ${deletes.length}');
+  if (verbose && deletes.isNotEmpty) {
+    print('[VERBOSE] deletes: $deletes');
+  }
+
+  await _writeToFile(file, newRulesText);
+
+  return (inserts: inserts, deletes: deletes);
+}
+
+Future<Diff> _readFile(File file, bool verbose) async {
+  final lines = utf8.decoder.bind(file.openRead()).transform(LineSplitter());
+  final set = <String>{};
+  await for (final line in lines) {
+    set.add(line);
+  }
+  if (verbose) {
+    print('[VERBOSE] oldValue: $set');
+  }
+  return set;
+}
+
+Future<String> _scrape({
+  required Uri uri,
+  required String querySelector,
+  required bool verbose,
+}) async {
+  // scraping HTML.
+  final response = await http.get(uri);
+  if (response.statusCode != 200) {
+    stderr.writeln('Failed to get HTML: ${response.statusCode}');
+    exit(1);
+  }
+  final document = parse(response.body);
+
+  final element = document.querySelectorAll(querySelector).firstOrNull?.text;
+  if (element == null) {
+    stderr.writeln('No elements found.');
+    exit(1);
+  }
+  if (verbose) {
+    print('[VERBOSE] scraped element: $element');
+  }
+  return element;
+}
+
+Future<void> _writeToFile(File file, String value) async {
+  await file.writeAsString('$value\n', mode: FileMode.writeOnly);
+}

packages/diffscrape/lib/src/arg.dart

@@ -0,0 +1,51 @@
+import 'package:args/args.dart';
+
+const argUri = 'uri';
+const argFile = 'file';
+const argQuerySelector = 'query-selector';
+const argFilteringPrefix = 'filtering-prefix';
+
+ArgParser buildParser() {
+  return ArgParser()
+    ..addOption(
+      argUri,
+      abbr: 'u',
+      mandatory: true,
+      help: 'URI to scrape.',
+    )
+    ..addOption(
+      argFile,
+      abbr: 'f',
+      mandatory: true,
+      help: 'Path to the file to compare against.',
+    )
+    ..addOption(
+      argQuerySelector,
+      abbr: 'q',
+      mandatory: true,
+      help: 'HTML query selector for the result.',
+    )
+    ..addOption(
+      argFilteringPrefix,
+      abbr: 'p',
+      help: 'Filtering prefix for the result.',
+    )
+    // common flags.
+    ..addFlag(
+      'help',
+      abbr: 'h',
+      negatable: false,
+      help: 'Print this usage information.',
+    )
+    ..addFlag(
+      'verbose',
+      abbr: 'v',
+      negatable: false,
+      help: 'Show additional command output.',
+    )
+    ..addFlag(
+      'version',
+      negatable: false,
+      help: 'Print the tool version.',
+    );
+}

packages/diffscrape/lib/src/util.dart

@@ -0,0 +1,6 @@
+import 'package:args/args.dart';
+
+void printUsage(ArgParser argParser) {
+  print('Usage: dart diffscrape.dart <flags> [arguments]');
+  print(argParser.usage);
+}