Add link preservation for GitHub and Pivotal links

.github/workflows/lint-pivotal-import.yaml

@@ -20,13 +20,10 @@ jobs:
       - name: Check out repository code
         uses: actions/checkout@v4
 
-      - name: Get Python Version
-        run: echo "PYTHON_VERSION=$(cat .python-version | grep ^[^#])" >> $GITHUB_ENV
-
-      - name: Set up Python ${{ env.PYTHON_VERSION }}
+      - name: Set up Python 3.10
         uses: actions/setup-python@v4
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: "3.10"
           cache: "pipenv"
 
       - name: Install pipenv

.github/workflows/test-pivotal-import.yaml

@@ -20,13 +20,10 @@ jobs:
       - name: Check out repository code
         uses: actions/checkout@v4
 
-      - name: Get Python Version
-        run: echo "PYTHON_VERSION=$(cat .python-version | grep ^[^#])" >> $GITHUB_ENV
-
-      - name: Set up Python ${{ env.PYTHON_VERSION }}
+      - name: Set up Python 3.10
         uses: actions/setup-python@v4
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: "3.10"
           cache: "pipenv"
 
       - name: Install pipenv

.gitignore

@@ -0,0 +1,5 @@
+# Ignore pipenv default cookbook folder
+cookbook/
+
+# Python dev
+.env

pivotal-import/.gitignore

@@ -3,3 +3,5 @@ config.json
 
 # Python
 .coverage
+.env
+__pycache__

pivotal-import/lib.py

@@ -22,6 +22,36 @@
 # Logging
 logger = logging.getLogger(__name__)
 
+
+# HTTP / Requests Exception Handling
+# https://github.com/useshortcut/api-cookbook/issues/83
+#
+# When a non 200 status code is received, give the users information about
+# the request and failure
+#
+def requests_error_decorator(func):
+    def inner_function(*args, **kwargs):
+        try:
+            func(*args, **kwargs)
+        except requests.exceptions.HTTPError as errh:
+            error_message = f"\nHTTP Error:\n"
+            error_message += f" Code: {errh.response.status_code}\n"
+            error_message += f" Url:  {errh.request.url}\n"
+            error_message += f"\nServer Response:\n   {errh.response.json()}"
+            raise requests.exceptions.HTTPError(error_message) from errh
+        except requests.exceptions.ConnectionError as errc:
+            print(f"Error Connecting: {errc}")
+            raise requests.exceptions.HTTPError(error_message) from errc
+        except requests.exceptions.Timeout as errt:
+            print(f"Timeout Error: {errt}")
+            raise requests.exceptions.HTTPError(error_message) from errt
+        except requests.exceptions.RequestException as err:
+            print(f"Unknown Http Error: {err}")
+            raise requests.exceptions.HTTPError(error_message) from err
+
+    return inner_function
+
+
 # Rate limiting. See https://developer.shortcut.com/api/rest/v3#Rate-Limiting
 # The Shortcut API limit is 200 per minute; the 200th request within 60 seconds
 # will receive an HTTP 429 response.
@@ -65,6 +95,7 @@ def print_rate_limiting_explanation():
 
 
 @rate_decorator(rate_mapping)
+@requests_error_decorator
 def sc_get(path, params={}):
     """
     Make a GET api call.
@@ -75,10 +106,12 @@ def sc_get(path, params={}):
     logger.debug("GET url=%s params=%s headers=%s" % (url, params, headers))
     resp = requests.get(url, headers=headers, params=params)
     resp.raise_for_status()
+    logger.debug("Server Response: %s" % (resp.content))
     return resp.json()
 
 
 @rate_decorator(rate_mapping)
+@requests_error_decorator
 def sc_post(path, data={}):
     """Make a POST api call.
 
@@ -97,6 +130,7 @@ def sc_post(path, data={}):
 
 
 @rate_decorator(rate_mapping)
+@requests_error_decorator
 def sc_put(path, data={}):
     """
     Make a PUT api call.
@@ -112,6 +146,7 @@ def sc_put(path, data={}):
 
 
 @rate_decorator(rate_mapping)
+@requests_error_decorator
 def sc_upload_files(files):
     """Upload and associate `files` with the story with given `story_id`"""
     url = f"{api_url_base}/files"
@@ -142,6 +177,7 @@ def sc_upload_files(files):
 
 
 @rate_decorator(rate_mapping)
+@requests_error_decorator
 def sc_delete(path):
     """
     Make a DELETE api call.

pivotal-import/pivotal_import.py

@@ -84,8 +84,50 @@ def create_stories(stories):
     return items
 
 
+def transform_pivotal_link(text, ctx):
+    """Transform Pivotal Tracker links to Shortcut story links.
+
+    Args:
+        text: Text containing Pivotal Tracker links
+        ctx: Context dictionary containing ID mappings
+
+    Returns:
+        str: Text with transformed links
+    """
+    # Transform full URLs
+    url_pattern = r'https://www\.pivotaltracker\.com/story/show/(\d+)'
+    def url_replace(match):
+        pt_id = match.group(1)
+        sc_id = ctx.get("id_mapping", {}).get(pt_id)
+        return f'https://app.shortcut.com/shortcut/story/{sc_id if sc_id else pt_id}'
+    text = re.sub(url_pattern, url_replace, text)
+
+    # Transform ID references (#123)
+    id_pattern = r'#(\d+)'
+    def id_replace(match):
+        pt_id = match.group(1)
+        sc_id = ctx.get("id_mapping", {}).get(pt_id)
+        return f'[{sc_id}]' if sc_id else f'#{pt_id}'
+    return re.sub(id_pattern, id_replace, text)
+
+def transform_github_link(url):
+    """Transform GitHub PR/branch links to external link format.
+
+    Args:
+        url (str): GitHub URL for PR or branch
+
+    Returns:
+        str: Standardized external link format
+    """
+    # Already in standard format - GitHub URLs work as-is
+    return url
+
 def url_to_external_links(url):
-    return [url]
+    if "pivotaltracker.com" in url:
+        return []  # Skip Pivotal links - they'll be transformed in text
+    if "github.com" in url:
+        return [transform_github_link(url)]
+    return [url]  # Preserve existing behavior for other URLs
 
 
 def parse_labels(labels: str):
@@ -177,7 +219,7 @@ def escape_md_table_syntax(s):
     return s.replace("|", "\\|")
 
 
-def parse_row(row, headers):
+def parse_row(row, headers, ctx=None):
     d = dict()
     for ix, val in enumerate(row):
         v = val.strip()
@@ -188,10 +230,17 @@ def parse_row(row, headers):
         if col in col_map:
             col_info = col_map[col]
             if isinstance(col_info, str):
+                if col == "description" and ctx and "id_mapping" in ctx:
+                    # Transform Pivotal links in description
+                    v = transform_pivotal_link(v, ctx)
                 d[col_info] = v
             else:
                 (key, translator) = col_info
-                d[key] = translator(v)
+                if col == "url":
+                    # URL field uses url_to_external_links translator
+                    d[key] = translator(v)
+                else:
+                    d[key] = translator(v)
 
         if col in nested_col_map:
             col_info = nested_col_map[col]
@@ -202,6 +251,11 @@ def parse_row(row, headers):
                 (key, translator) = col_info
                 v = translator(v)
             d.setdefault(key, []).append(v)
+
+        # Handle GitHub PR/branch links
+        if col in ["pull_request", "git_branch"] and v:
+            d.setdefault("external_links", []).append(transform_github_link(v))
+
     return d
 
 
@@ -594,9 +648,28 @@ def process_pt_csv_export(ctx, pt_csv_file, entity_collector):
     with open(pt_csv_file) as csvfile:
         reader = csv.reader(csvfile)
         header = [col.lower() for col in next(reader)]
+
+        # First pass: collect all stories to build ID mapping
+        story_rows = []
         for row in reader:
-            row_info = parse_row(row, header)
+            story_rows.append(row)
+            row_info = parse_row(row, header, ctx)
+            if "id" in row_info:
+                ctx["id_mapping"][row_info["id"]] = None  # Placeholder for Shortcut ID
+
+        # Reset file pointer for second pass
+        csvfile.seek(0)
+        next(reader)  # Skip header
+
+        # Second pass: process stories with complete mapping
+        for row in story_rows:
+            row_info = parse_row(row, header, ctx)
             entity = build_entity(ctx, row_info)
+            if entity["type"] == "story":
+                # Update ID mapping with new Shortcut ID
+                pt_id = entity["parsed_row"]["id"]
+                if "imported_entity" in entity:
+                    ctx["id_mapping"][pt_id] = entity["imported_entity"]["id"]
             logger.debug("Emitting Entity: %s", entity)
             stats.update(entity_collector.collect(entity))
 
@@ -632,6 +705,7 @@ def build_ctx(cfg):
         "priority_custom_field_id": cfg["priority_custom_field_id"],
         "user_config": load_users(cfg["users_csv_file"]),
         "workflow_config": load_workflow_states(cfg["states_csv_file"]),
+        "id_mapping": {},  # Initialize empty mapping for Pivotal->Shortcut IDs
     }
     logger.debug("Built context %s", ctx)
     return ctx

pivotal-import/pivotal_import_test.py

@@ -18,10 +18,12 @@ def create_test_ctx():
 
 
 def test_parse_row_basic():
+    """Test basic row parsing without link transformation."""
+    # No ctx provided, so no link transformation should occur
     assert {
         "name": "My Story Name",
         "description": "My Story Description",
-    } == parse_row(["My Story Name", "My Story Description"], ["title", "description"])
+    } == parse_row(["My Story Name", "My Story Description"], ["title", "description"], None)
 
 
 def test_parse_comments():
@@ -656,7 +658,7 @@ def test_entity_collector_with_epics():
         }
     )
 
-    # When: the entities are commited/crread
+    # When: the entities are committed/created
     created = entity_collector.commit()
 
     # Then: All epics are created before the stories
@@ -702,3 +704,71 @@ def test_entity_collector_with_epics():
             "external_id": "3456",
         },
     ] == created
+
+
+def test_transform_pivotal_link():
+    """Test Pivotal Tracker link transformation."""
+    ctx = create_test_ctx()
+    ctx["id_mapping"] = {"12345": "sc-789", "67890": "sc-012"}
+
+    # Test URL format
+    assert transform_pivotal_link(
+        "https://www.pivotaltracker.com/story/show/12345",
+        ctx
+    ) == "https://app.shortcut.com/shortcut/story/sc-789"
+
+    # Test ID reference format
+    assert transform_pivotal_link("#12345", ctx) == "[sc-789]"
+
+    # Test mixed content
+    text = "See #12345 and https://www.pivotaltracker.com/story/show/67890 for details"
+    expected = "See [sc-789] and https://app.shortcut.com/shortcut/story/sc-012 for details"
+    assert transform_pivotal_link(text, ctx) == expected
+
+    # Test unmapped ID
+    assert transform_pivotal_link("#99999", ctx) == "#99999"
+
+    # Test unmapped URL
+    assert transform_pivotal_link(
+        "https://www.pivotaltracker.com/story/show/99999",
+        ctx
+    ) == "https://app.shortcut.com/shortcut/story/99999"
+
+
+def test_transform_github_link():
+    """Test GitHub link transformation."""
+    # Test PR link format
+    assert transform_github_link(
+        "https://github.com/org/repo/pull/123"
+    ) == "https://github.com/org/repo/pull/123"
+
+    # Test branch link format
+    assert transform_github_link(
+        "https://github.com/org/repo/tree/feature-branch"
+    ) == "https://github.com/org/repo/tree/feature-branch"
+
+    # Test other GitHub links (should remain unchanged)
+    assert transform_github_link(
+        "https://github.com/org/repo/issues/456"
+    ) == "https://github.com/org/repo/issues/456"
+
+
+def test_url_to_external_links():
+    """Test URL to external links conversion."""
+    ctx = create_test_ctx()
+
+    # Test Pivotal links (should be skipped)
+    assert url_to_external_links(
+        "https://www.pivotaltracker.com/story/show/12345"
+    ) == []
+
+    # Test GitHub links
+    github_pr = "https://github.com/org/repo/pull/123"
+    assert url_to_external_links(github_pr) == [github_pr]
+
+    github_branch = "https://github.com/org/repo/tree/feature-branch"
+    assert url_to_external_links(github_branch) == [github_branch]
+
+    # Test other external links
+    other_link = "https://example.com/page"
+    assert url_to_external_links(other_link) == [other_link]