Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Caveat failure on group-by attribute #13

Open
okennedy opened this issue Oct 9, 2023 · 1 comment
Open

Caveat failure on group-by attribute #13

okennedy opened this issue Oct 9, 2023 · 1 comment

Comments

@okennedy
Copy link
Member

okennedy commented Oct 9, 2023

Describe the bug
Caveat failure on group-by aggregate dataset

Specifically note the difference in exprIds between the outer project and the first aggregate:
idx#261443 in the former, idx#262323 in the latter.

+- !Project [grp#261406, idx#261443, (__MIMIR_ATTR_CAVEATTED__grp_261406#262188 OR __MIMIR_ALL_GROUPS_CONTAMINATED#262190) AS __MIMIR_ATTR_CAVEATTED__grp_261406#262193, (__MIMIR_ATTR_CAVEATTED__idx_261443#262189 OR __MIMIR_ALL_GROUPS_CONTAMINATED#262190) AS __MIMIR_ATTR_CAVEATTED__idx_261443#262194, __MIMIR_ROW_CAVEATTED#262187 AS __MIMIR_ROW_CAVEATTED#262192]
  +- Join Cross
     :- Aggregate [idx#262323], [min(idx#262319) AS grp#261406, idx#262323, ((sum(if (__MIMIR_ATTR_CAVEATTED__idx_261407#262171) 1 else 0) > cast(0 as bigint)) OR (sum(if ((__MIMIR_ATTR_CAVEATTED__idx_261443#262176 OR __MIMIR_ROW_CAVEATTED#262170)) 1 else 0) > cast(0 as bigint))) AS __MIMIR_ATTR_CAVEATTED__grp_261406#262188, ((sum(if (__MIMIR_ATTR_CAVEATTED__idx_261443#262176) 1 else 0) > cast(0 as bigint)) OR (sum(if (__MIMIR_ATTR_CAVEATTED__idx_261443#262176) 1 else 0) > cast(0 as bigint))) AS __MIMIR_ATTR_CAVEATTED__idx_261443#262189, NOT (sum(if ((NOT __MIMIR_ROW_CAVEATTED#262170 AND NOT __MIMIR_ATTR_CAVEATTED__idx_261443#262176)) 1 else 0) > cast(0 as bigint)) AS __MIMIR_ROW_CAVEATTED#262187]
     :  +- Project [idx#262319, file#262320, lat#262321, lon#262322, geometry#261430, idx#262323, file#262324, lat#262325, lon#262326, geometry#261461, __MIMIR_ROWID#261861L, __MIMIR_ATTR_CAVEATTED__idx_261407#262159 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262171, __MIMIR_ATTR_CAVEATTED__file_261408#262160 AS __MIMIR_ATTR_CAVEATTED__file_261408#262172, __MIMIR_ATTR_CAVEATTED__lat_261409#262161 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262173, __MIMIR_ATTR_CAVEATTED__lon_261410#262162 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262174, __MIMIR_ATTR_CAVEATTED__geometry_261430#262163 AS __MIMIR_ATTR_CAVEATTED__geometry_261430#262175, __MIMIR_ATTR_CAVEATTED__idx_261443#262164 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262176, __MIMIR_ATTR_CAVEATTED__file_261444#262165 AS __MIMIR_ATTR_CAVEATTED__file_261444#262177, __MIMIR_ATTR_CAVEATTED__lat_261445#262166 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262178, __MIMIR_ATTR_CAVEATTED__lon_261446#262167 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262179, __MIMIR_ATTR_CAVEATTED__geometry_261461#262168 AS __MIMIR_ATTR_CAVEATTED__geometry_261461#262180, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261861#262169 AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261861#262181, (__MIMIR_ROW_CAVEATTED#262158 OR (__MIMIR_ATTR_CAVEATTED__geometry_261430#262163 OR __MIMIR_ATTR_CAVEATTED__geometry_261461#262168)) AS __MIMIR_ROW_CAVEATTED#262170]
     :     +- Filter ( **org.apache.spark.sql.sedona_sql.expressions.ST_Distance$**   < cast(0.015 as double))
     :        +- Project [idx#262319, file#262320, lat#262321, lon#262322, geometry#261430, idx#262323, file#262324, lat#262325, lon#262326, geometry#261461, __MIMIR_ROWID#261861L, __MIMIR_ATTR_CAVEATTED__idx_261407#262145 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262159, __MIMIR_ATTR_CAVEATTED__file_261408#262146 AS __MIMIR_ATTR_CAVEATTED__file_261408#262160, __MIMIR_ATTR_CAVEATTED__lat_261409#262147 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262161, __MIMIR_ATTR_CAVEATTED__lon_261410#262148 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262162, __MIMIR_ATTR_CAVEATTED__geometry_261430#262149 AS __MIMIR_ATTR_CAVEATTED__geometry_261430#262163, __MIMIR_ATTR_CAVEATTED__idx_261443#262151 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262164, __MIMIR_ATTR_CAVEATTED__file_261444#262152 AS __MIMIR_ATTR_CAVEATTED__file_261444#262165, __MIMIR_ATTR_CAVEATTED__lat_261445#262153 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262166, __MIMIR_ATTR_CAVEATTED__lon_261446#262154 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262167, __MIMIR_ATTR_CAVEATTED__geometry_261461#262155 AS __MIMIR_ATTR_CAVEATTED__geometry_261461#262168, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261861#262157 AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261861#262169, __MIMIR_ROW_CAVEATTED#262144 AS __MIMIR_ROW_CAVEATTED#262158]
     :           +- Project [idx#262319, file#262320, lat#262321, lon#262322, geometry#261430, LHS___MIMIR_ROWID#261859L, idx#262323, file#262324, lat#262325, lon#262326, geometry#261461, RHS___MIMIR_ROWID#261860L, cast(hash(ifnull(LHS___MIMIR_ROWID#261859L, 1), ifnull(RHS___MIMIR_ROWID#261860L, 1), 42) as bigint) AS __MIMIR_ROWID#261861L, __MIMIR_ATTR_CAVEATTED__idx_261407#262132 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262145, __MIMIR_ATTR_CAVEATTED__file_261408#262133 AS __MIMIR_ATTR_CAVEATTED__file_261408#262146, __MIMIR_ATTR_CAVEATTED__lat_261409#262134 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262147, __MIMIR_ATTR_CAVEATTED__lon_261410#262135 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262148, __MIMIR_ATTR_CAVEATTED__geometry_261430#262136 AS __MIMIR_ATTR_CAVEATTED__geometry_261430#262149, __MIMIR_ATTR_CAVEATTED__LHS___MIMIR_ROWID_261859#262137 AS __MIMIR_ATTR_CAVEATTED__LHS___MIMIR_ROWID_261859#262150, __MIMIR_ATTR_CAVEATTED__idx_261443#262138 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262151, __MIMIR_ATTR_CAVEATTED__file_261444#262139 AS __MIMIR_ATTR_CAVEATTED__file_261444#262152, __MIMIR_ATTR_CAVEATTED__lat_261445#262140 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262153, __MIMIR_ATTR_CAVEATTED__lon_261446#262141 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262154, __MIMIR_ATTR_CAVEATTED__geometry_261461#262142 AS __MIMIR_ATTR_CAVEATTED__geometry_261461#262155, ... 3 more fields]
     :              +- Project [idx#262319, file#262320, lat#262321, lon#262322, geometry#261430, LHS___MIMIR_ROWID#261859L, idx#262323, file#262324, lat#262325, lon#262326, geometry#261461, RHS___MIMIR_ROWID#261860L, __MIMIR_ATTR_CAVEATTED__idx_261407#262086 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262132, __MIMIR_ATTR_CAVEATTED__file_261408#262087 AS __MIMIR_ATTR_CAVEATTED__file_261408#262133, __MIMIR_ATTR_CAVEATTED__lat_261409#262088 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262134, __MIMIR_ATTR_CAVEATTED__lon_261410#262089 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262135, __MIMIR_ATTR_CAVEATTED__geometry_261430#262090 AS __MIMIR_ATTR_CAVEATTED__geometry_261430#262136, __MIMIR_ATTR_CAVEATTED__LHS___MIMIR_ROWID_261859#262091 AS __MIMIR_ATTR_CAVEATTED__LHS___MIMIR_ROWID_261859#262137, __MIMIR_ATTR_CAVEATTED__idx_261443#262125 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262138, __MIMIR_ATTR_CAVEATTED__file_261444#262126 AS __MIMIR_ATTR_CAVEATTED__file_261444#262139, __MIMIR_ATTR_CAVEATTED__lat_261445#262127 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262140, __MIMIR_ATTR_CAVEATTED__lon_261446#262128 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262141, __MIMIR_ATTR_CAVEATTED__geometry_261461#262129 AS __MIMIR_ATTR_CAVEATTED__geometry_261461#262142, __MIMIR_ATTR_CAVEATTED__RHS___MIMIR_ROWID_261860#262130 AS __MIMIR_ATTR_CAVEATTED__RHS___MIMIR_ROWID_261860#262143, (__MIMIR_ROW_CAVEATTED#262085 OR __MIMIR_ROW_CAVEATTED#262124) AS __MIMIR_ROW_CAVEATTED#262131]
     :                 +- Join Inner
     :                    :- Project [idx#262319, file#262320, lat#262321, lon#262322, geometry#261430, __MIMIR_ROWID#261601L AS LHS___MIMIR_ROWID#261859L, __MIMIR_ATTR_CAVEATTED__idx_261407#262079 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262086, __MIMIR_ATTR_CAVEATTED__file_261408#262080 AS __MIMIR_ATTR_CAVEATTED__file_261408#262087, __MIMIR_ATTR_CAVEATTED__lat_261409#262081 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262088, __MIMIR_ATTR_CAVEATTED__lon_261410#262082 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262089, __MIMIR_ATTR_CAVEATTED__geometry_261430#262083 AS __MIMIR_ATTR_CAVEATTED__geometry_261430#262090, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261601#262084 AS __MIMIR_ATTR_CAVEATTED__LHS___MIMIR_ROWID_261859#262091, __MIMIR_ROW_CAVEATTED#262078 AS __MIMIR_ROW_CAVEATTED#262085]
     :                    :  +- Project [idx#262319, file#262320, lat#262321, lon#262322, CASE WHEN (isnull(lon#262322) OR isnull(lat#262321)) THEN cast(null as geometry) ELSE st_point(cast(lon#262322 as decimal(24,20)), cast(lat#262321 as decimal(24,20))) END AS geometry#261430, __MIMIR_ROWID#261601L, __MIMIR_ATTR_CAVEATTED__idx_261407#262072 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262079, __MIMIR_ATTR_CAVEATTED__file_261408#262073 AS __MIMIR_ATTR_CAVEATTED__file_261408#262080, __MIMIR_ATTR_CAVEATTED__lat_261409#262074 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262081, __MIMIR_ATTR_CAVEATTED__lon_261410#262075 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262082, CASE WHEN (((__MIMIR_ATTR_CAVEATTED__lon_261410#262075 AND __MIMIR_ATTR_CAVEATTED__lat_261409#262074) OR (__MIMIR_ATTR_CAVEATTED__lon_261410#262075 AND NOT isnull(lat#262321))) OR (__MIMIR_ATTR_CAVEATTED__lat_261409#262074 AND NOT isnull(lon#262322))) THEN true WHEN (isnull(lon#262322) OR isnull(lat#262321)) THEN false ELSE (__MIMIR_ATTR_CAVEATTED__lon_261410#262075 OR __MIMIR_ATTR_CAVEATTED__lat_261409#262074) END AS __MIMIR_ATTR_CAVEATTED__geometry_261430#262083, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261601#262077 AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261601#262084, __MIMIR_ROW_CAVEATTED#262071 AS __MIMIR_ROW_CAVEATTED#262078]
     :                    :     +- Project [idx#262319, file#262320, lat#262321, lon#262322, cast(null as geometry) AS geometry#261420, __MIMIR_ROWID#261601L, __MIMIR_ATTR_CAVEATTED__idx_261407#262066 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262072, __MIMIR_ATTR_CAVEATTED__file_261408#262067 AS __MIMIR_ATTR_CAVEATTED__file_261408#262073, __MIMIR_ATTR_CAVEATTED__lat_261409#262068 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262074, __MIMIR_ATTR_CAVEATTED__lon_261410#262069 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262075, false AS __MIMIR_ATTR_CAVEATTED__geometry_261420#262076, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261601#262070 AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261601#262077, __MIMIR_ROW_CAVEATTED#262065 AS __MIMIR_ROW_CAVEATTED#262071]
     :                    :        +- Project [idx#262319, file#262320, lat#262321, lon#262322, cast(hash(if (isnull(__MIMIR_ROWID_PARTITION_ID#261602L)) cast(__MIMIR_ROWID_INTERNAL_ID#261604L as bigint) else (cast(__MIMIR_ROWID_INTERNAL_ID#261604L as bigint) + cast(if (isnull(cast(__MIMIR_ROWID_PARTITION_ID#261602L as int))) null else FIRST_IDENTIFIER_FOR_PARTITION(knownnotnull(cast(__MIMIR_ROWID_PARTITION_ID#261602L as int))) as bigint)), idx#262319, file#262320, lat#262321, lon#262322, 42) as bigint) AS __MIMIR_ROWID#261601L, __MIMIR_ATTR_CAVEATTED__idx_261407#262059 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262066, __MIMIR_ATTR_CAVEATTED__file_261408#262060 AS __MIMIR_ATTR_CAVEATTED__file_261408#262067, __MIMIR_ATTR_CAVEATTED__lat_261409#262061 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262068, __MIMIR_ATTR_CAVEATTED__lon_261410#262062 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262069, (((((__MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261602#262063 OR if (isnull(__MIMIR_ROWID_PARTITION_ID#261602L)) __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_INTERNAL_ID_261604#262064 else (__MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_INTERNAL_ID_261604#262064 OR (__MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261602#262063 OR if (isnull(cast(__MIMIR_ROWID_PARTITION_ID#261602L as int))) false else __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261602#262063))) OR __MIMIR_ATTR_CAVEATTED__idx_261407#262059) OR __MIMIR_ATTR_CAVEATTED__file_261408#262060) OR __MIMIR_ATTR_CAVEATTED__lat_261409#262061) OR __MIMIR_ATTR_CAVEATTED__lon_261410#262062) AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261601#262070, __MIMIR_ROW_CAVEATTED#262058 AS __MIMIR_ROW_CAVEATTED#262065]
     :                    :           +- Project [idx#262319, file#262320, lat#262321, lon#262322, cast(SPARK_PARTITION_ID() as bigint) AS __MIMIR_ROWID_PARTITION_ID#261602L, cast(monotonically_increasing_id() as bigint) AS __MIMIR_ROWID_INTERNAL_ID#261604L, __MIMIR_ATTR_CAVEATTED__idx_261407#262054 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262059, __MIMIR_ATTR_CAVEATTED__file_261408#262055 AS __MIMIR_ATTR_CAVEATTED__file_261408#262060, __MIMIR_ATTR_CAVEATTED__lat_261409#262056 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262061, __MIMIR_ATTR_CAVEATTED__lon_261410#262057 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262062, false AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261602#262063, false AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_INTERNAL_ID_261604#262064, __MIMIR_ROW_CAVEATTED#262053 AS __MIMIR_ROW_CAVEATTED#262058]
     :                    :              +- Project [idx#262319, file#262320, lat#262321, lon#262322, false AS __MIMIR_ATTR_CAVEATTED__idx_261407#262054, false AS __MIMIR_ATTR_CAVEATTED__file_261408#262055, false AS __MIMIR_ATTR_CAVEATTED__lat_261409#262056, false AS __MIMIR_ATTR_CAVEATTED__lon_261410#262057, false AS __MIMIR_ROW_CAVEATTED#262053]
     :                    :                 +- LocalRelation [idx#262319, file#262320, lat#262321, lon#262322]
     :                    +- Project [idx#262323, file#262324, lat#262325, lon#262326, geometry#261461, __MIMIR_ROWID#261778L AS RHS___MIMIR_ROWID#261860L, __MIMIR_ATTR_CAVEATTED__idx_261443#262118 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262125, __MIMIR_ATTR_CAVEATTED__file_261444#262119 AS __MIMIR_ATTR_CAVEATTED__file_261444#262126, __MIMIR_ATTR_CAVEATTED__lat_261445#262120 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262127, __MIMIR_ATTR_CAVEATTED__lon_261446#262121 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262128, __MIMIR_ATTR_CAVEATTED__geometry_261461#262122 AS __MIMIR_ATTR_CAVEATTED__geometry_261461#262129, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261778#262123 AS __MIMIR_ATTR_CAVEATTED__RHS___MIMIR_ROWID_261860#262130, __MIMIR_ROW_CAVEATTED#262117 AS __MIMIR_ROW_CAVEATTED#262124]
     :                       +- Project [idx#262323, file#262324, lat#262325, lon#262326, CASE WHEN (isnull(lon#262326) OR isnull(lat#262325)) THEN cast(null as geometry) ELSE st_point(cast(lon#262326 as decimal(24,20)), cast(lat#262325 as decimal(24,20))) END AS geometry#261461, __MIMIR_ROWID#261778L, __MIMIR_ATTR_CAVEATTED__idx_261443#262111 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262118, __MIMIR_ATTR_CAVEATTED__file_261444#262112 AS __MIMIR_ATTR_CAVEATTED__file_261444#262119, __MIMIR_ATTR_CAVEATTED__lat_261445#262113 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262120, __MIMIR_ATTR_CAVEATTED__lon_261446#262114 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262121, CASE WHEN (((__MIMIR_ATTR_CAVEATTED__lon_261446#262114 AND __MIMIR_ATTR_CAVEATTED__lat_261445#262113) OR (__MIMIR_ATTR_CAVEATTED__lon_261446#262114 AND NOT isnull(lat#262325))) OR (__MIMIR_ATTR_CAVEATTED__lat_261445#262113 AND NOT isnull(lon#262326))) THEN true WHEN (isnull(lon#262326) OR isnull(lat#262325)) THEN false ELSE (__MIMIR_ATTR_CAVEATTED__lon_261446#262114 OR __MIMIR_ATTR_CAVEATTED__lat_261445#262113) END AS __MIMIR_ATTR_CAVEATTED__geometry_261461#262122, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261778#262116 AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261778#262123, __MIMIR_ROW_CAVEATTED#262110 AS __MIMIR_ROW_CAVEATTED#262117]
     :                          +- Project [idx#262323, file#262324, lat#262325, lon#262326, cast(null as geometry) AS geometry#261453, __MIMIR_ROWID#261778L, __MIMIR_ATTR_CAVEATTED__idx_261443#262105 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262111, __MIMIR_ATTR_CAVEATTED__file_261444#262106 AS __MIMIR_ATTR_CAVEATTED__file_261444#262112, __MIMIR_ATTR_CAVEATTED__lat_261445#262107 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262113, __MIMIR_ATTR_CAVEATTED__lon_261446#262108 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262114, false AS __MIMIR_ATTR_CAVEATTED__geometry_261453#262115, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261778#262109 AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261778#262116, __MIMIR_ROW_CAVEATTED#262104 AS __MIMIR_ROW_CAVEATTED#262110]
     :                             +- Project [idx#262323, file#262324, lat#262325, lon#262326, cast(hash(if (isnull(__MIMIR_ROWID_PARTITION_ID#261779L)) cast(__MIMIR_ROWID_INTERNAL_ID#261781L as bigint) else (cast(__MIMIR_ROWID_INTERNAL_ID#261781L as bigint) + cast(if (isnull(cast(__MIMIR_ROWID_PARTITION_ID#261779L as int))) null else FIRST_IDENTIFIER_FOR_PARTITION(knownnotnull(cast(__MIMIR_ROWID_PARTITION_ID#261779L as int))) as bigint)), idx#262323, file#262324, lat#262325, lon#262326, 42) as bigint) AS __MIMIR_ROWID#261778L, __MIMIR_ATTR_CAVEATTED__idx_261443#262098 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262105, __MIMIR_ATTR_CAVEATTED__file_261444#262099 AS __MIMIR_ATTR_CAVEATTED__file_261444#262106, __MIMIR_ATTR_CAVEATTED__lat_261445#262100 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262107, __MIMIR_ATTR_CAVEATTED__lon_261446#262101 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262108, (((((__MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261779#262102 OR if (isnull(__MIMIR_ROWID_PARTITION_ID#261779L)) __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_INTERNAL_ID_261781#262103 else (__MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_INTERNAL_ID_261781#262103 OR (__MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261779#262102 OR if (isnull(cast(__MIMIR_ROWID_PARTITION_ID#261779L as int))) false else __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261779#262102))) OR __MIMIR_ATTR_CAVEATTED__idx_261443#262098) OR __MIMIR_ATTR_CAVEATTED__file_261444#262099) OR __MIMIR_ATTR_CAVEATTED__lat_261445#262100) OR __MIMIR_ATTR_CAVEATTED__lon_261446#262101) AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261778#262109, __MIMIR_ROW_CAVEATTED#262097 AS __MIMIR_ROW_CAVEATTED#262104]
     :                                +- Project [idx#262323, file#262324, lat#262325, lon#262326, cast(SPARK_PARTITION_ID() as bigint) AS __MIMIR_ROWID_PARTITION_ID#261779L, cast(monotonically_increasing_id() as bigint) AS __MIMIR_ROWID_INTERNAL_ID#261781L, __MIMIR_ATTR_CAVEATTED__idx_261443#262093 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262098, __MIMIR_ATTR_CAVEATTED__file_261444#262094 AS __MIMIR_ATTR_CAVEATTED__file_261444#262099, __MIMIR_ATTR_CAVEATTED__lat_261445#262095 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262100, __MIMIR_ATTR_CAVEATTED__lon_261446#262096 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262101, false AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261779#262102, false AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_INTERNAL_ID_261781#262103, __MIMIR_ROW_CAVEATTED#262092 AS __MIMIR_ROW_CAVEATTED#262097]
     :                                   +- Project [idx#262323, file#262324, lat#262325, lon#262326, false AS __MIMIR_ATTR_CAVEATTED__idx_261443#262093, false AS __MIMIR_ATTR_CAVEATTED__file_261444#262094, false AS __MIMIR_ATTR_CAVEATTED__lat_261445#262095, false AS __MIMIR_ATTR_CAVEATTED__lon_261446#262096, false AS __MIMIR_ROW_CAVEATTED#262092]
     :                                      +- LocalRelation [idx#262323, file#262324, lat#262325, lon#262326]
     +- Aggregate [(sum(if (__MIMIR_ATTR_CAVEATTED__idx_261443#262176) 1 else 0) > cast(0 as bigint)) AS __MIMIR_ALL_GROUPS_CONTAMINATED#262190]
        +- Project [idx#262327, file#262328, lat#262329, lon#262330, geometry#261430, idx#262331, file#262332, lat#262333, lon#262334, geometry#261461, __MIMIR_ROWID#261861L, __MIMIR_ATTR_CAVEATTED__idx_261407#262159 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262171, __MIMIR_ATTR_CAVEATTED__file_261408#262160 AS __MIMIR_ATTR_CAVEATTED__file_261408#262172, __MIMIR_ATTR_CAVEATTED__lat_261409#262161 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262173, __MIMIR_ATTR_CAVEATTED__lon_261410#262162 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262174, __MIMIR_ATTR_CAVEATTED__geometry_261430#262163 AS __MIMIR_ATTR_CAVEATTED__geometry_261430#262175, __MIMIR_ATTR_CAVEATTED__idx_261443#262164 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262176, __MIMIR_ATTR_CAVEATTED__file_261444#262165 AS __MIMIR_ATTR_CAVEATTED__file_261444#262177, __MIMIR_ATTR_CAVEATTED__lat_261445#262166 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262178, __MIMIR_ATTR_CAVEATTED__lon_261446#262167 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262179, __MIMIR_ATTR_CAVEATTED__geometry_261461#262168 AS __MIMIR_ATTR_CAVEATTED__geometry_261461#262180, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261861#262169 AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261861#262181, (__MIMIR_ROW_CAVEATTED#262158 OR (__MIMIR_ATTR_CAVEATTED__geometry_261430#262163 OR __MIMIR_ATTR_CAVEATTED__geometry_261461#262168)) AS __MIMIR_ROW_CAVEATTED#262170]
           +- Filter ( **org.apache.spark.sql.sedona_sql.expressions.ST_Distance$**   < cast(0.015 as double))
              +- Project [idx#262327, file#262328, lat#262329, lon#262330, geometry#261430, idx#262331, file#262332, lat#262333, lon#262334, geometry#261461, __MIMIR_ROWID#261861L, __MIMIR_ATTR_CAVEATTED__idx_261407#262145 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262159, __MIMIR_ATTR_CAVEATTED__file_261408#262146 AS __MIMIR_ATTR_CAVEATTED__file_261408#262160, __MIMIR_ATTR_CAVEATTED__lat_261409#262147 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262161, __MIMIR_ATTR_CAVEATTED__lon_261410#262148 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262162, __MIMIR_ATTR_CAVEATTED__geometry_261430#262149 AS __MIMIR_ATTR_CAVEATTED__geometry_261430#262163, __MIMIR_ATTR_CAVEATTED__idx_261443#262151 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262164, __MIMIR_ATTR_CAVEATTED__file_261444#262152 AS __MIMIR_ATTR_CAVEATTED__file_261444#262165, __MIMIR_ATTR_CAVEATTED__lat_261445#262153 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262166, __MIMIR_ATTR_CAVEATTED__lon_261446#262154 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262167, __MIMIR_ATTR_CAVEATTED__geometry_261461#262155 AS __MIMIR_ATTR_CAVEATTED__geometry_261461#262168, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261861#262157 AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261861#262169, __MIMIR_ROW_CAVEATTED#262144 AS __MIMIR_ROW_CAVEATTED#262158]
                 +- Project [idx#262327, file#262328, lat#262329, lon#262330, geometry#261430, LHS___MIMIR_ROWID#261859L, idx#262331, file#262332, lat#262333, lon#262334, geometry#261461, RHS___MIMIR_ROWID#261860L, cast(hash(ifnull(LHS___MIMIR_ROWID#261859L, 1), ifnull(RHS___MIMIR_ROWID#261860L, 1), 42) as bigint) AS __MIMIR_ROWID#261861L, __MIMIR_ATTR_CAVEATTED__idx_261407#262132 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262145, __MIMIR_ATTR_CAVEATTED__file_261408#262133 AS __MIMIR_ATTR_CAVEATTED__file_261408#262146, __MIMIR_ATTR_CAVEATTED__lat_261409#262134 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262147, __MIMIR_ATTR_CAVEATTED__lon_261410#262135 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262148, __MIMIR_ATTR_CAVEATTED__geometry_261430#262136 AS __MIMIR_ATTR_CAVEATTED__geometry_261430#262149, __MIMIR_ATTR_CAVEATTED__LHS___MIMIR_ROWID_261859#262137 AS __MIMIR_ATTR_CAVEATTED__LHS___MIMIR_ROWID_261859#262150, __MIMIR_ATTR_CAVEATTED__idx_261443#262138 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262151, __MIMIR_ATTR_CAVEATTED__file_261444#262139 AS __MIMIR_ATTR_CAVEATTED__file_261444#262152, __MIMIR_ATTR_CAVEATTED__lat_261445#262140 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262153, __MIMIR_ATTR_CAVEATTED__lon_261446#262141 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262154, __MIMIR_ATTR_CAVEATTED__geometry_261461#262142 AS __MIMIR_ATTR_CAVEATTED__geometry_261461#262155, ... 3 more fields]
                    +- Project [idx#262327, file#262328, lat#262329, lon#262330, geometry#261430, LHS___MIMIR_ROWID#261859L, idx#262331, file#262332, lat#262333, lon#262334, geometry#261461, RHS___MIMIR_ROWID#261860L, __MIMIR_ATTR_CAVEATTED__idx_261407#262086 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262132, __MIMIR_ATTR_CAVEATTED__file_261408#262087 AS __MIMIR_ATTR_CAVEATTED__file_261408#262133, __MIMIR_ATTR_CAVEATTED__lat_261409#262088 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262134, __MIMIR_ATTR_CAVEATTED__lon_261410#262089 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262135, __MIMIR_ATTR_CAVEATTED__geometry_261430#262090 AS __MIMIR_ATTR_CAVEATTED__geometry_261430#262136, __MIMIR_ATTR_CAVEATTED__LHS___MIMIR_ROWID_261859#262091 AS __MIMIR_ATTR_CAVEATTED__LHS___MIMIR_ROWID_261859#262137, __MIMIR_ATTR_CAVEATTED__idx_261443#262125 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262138, __MIMIR_ATTR_CAVEATTED__file_261444#262126 AS __MIMIR_ATTR_CAVEATTED__file_261444#262139, __MIMIR_ATTR_CAVEATTED__lat_261445#262127 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262140, __MIMIR_ATTR_CAVEATTED__lon_261446#262128 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262141, __MIMIR_ATTR_CAVEATTED__geometry_261461#262129 AS __MIMIR_ATTR_CAVEATTED__geometry_261461#262142, __MIMIR_ATTR_CAVEATTED__RHS___MIMIR_ROWID_261860#262130 AS __MIMIR_ATTR_CAVEATTED__RHS___MIMIR_ROWID_261860#262143, (__MIMIR_ROW_CAVEATTED#262085 OR __MIMIR_ROW_CAVEATTED#262124) AS __MIMIR_ROW_CAVEATTED#262131]
                       +- Join Inner
                          :- Project [idx#262327, file#262328, lat#262329, lon#262330, geometry#261430, __MIMIR_ROWID#261601L AS LHS___MIMIR_ROWID#261859L, __MIMIR_ATTR_CAVEATTED__idx_261407#262079 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262086, __MIMIR_ATTR_CAVEATTED__file_261408#262080 AS __MIMIR_ATTR_CAVEATTED__file_261408#262087, __MIMIR_ATTR_CAVEATTED__lat_261409#262081 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262088, __MIMIR_ATTR_CAVEATTED__lon_261410#262082 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262089, __MIMIR_ATTR_CAVEATTED__geometry_261430#262083 AS __MIMIR_ATTR_CAVEATTED__geometry_261430#262090, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261601#262084 AS __MIMIR_ATTR_CAVEATTED__LHS___MIMIR_ROWID_261859#262091, __MIMIR_ROW_CAVEATTED#262078 AS __MIMIR_ROW_CAVEATTED#262085]
                          :  +- Project [idx#262327, file#262328, lat#262329, lon#262330, CASE WHEN (isnull(lon#262330) OR isnull(lat#262329)) THEN cast(null as geometry) ELSE st_point(cast(lon#262330 as decimal(24,20)), cast(lat#262329 as decimal(24,20))) END AS geometry#261430, __MIMIR_ROWID#261601L, __MIMIR_ATTR_CAVEATTED__idx_261407#262072 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262079, __MIMIR_ATTR_CAVEATTED__file_261408#262073 AS __MIMIR_ATTR_CAVEATTED__file_261408#262080, __MIMIR_ATTR_CAVEATTED__lat_261409#262074 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262081, __MIMIR_ATTR_CAVEATTED__lon_261410#262075 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262082, CASE WHEN (((__MIMIR_ATTR_CAVEATTED__lon_261410#262075 AND __MIMIR_ATTR_CAVEATTED__lat_261409#262074) OR (__MIMIR_ATTR_CAVEATTED__lon_261410#262075 AND NOT isnull(lat#262329))) OR (__MIMIR_ATTR_CAVEATTED__lat_261409#262074 AND NOT isnull(lon#262330))) THEN true WHEN (isnull(lon#262330) OR isnull(lat#262329)) THEN false ELSE (__MIMIR_ATTR_CAVEATTED__lon_261410#262075 OR __MIMIR_ATTR_CAVEATTED__lat_261409#262074) END AS __MIMIR_ATTR_CAVEATTED__geometry_261430#262083, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261601#262077 AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261601#262084, __MIMIR_ROW_CAVEATTED#262071 AS __MIMIR_ROW_CAVEATTED#262078]
                          :     +- Project [idx#262327, file#262328, lat#262329, lon#262330, cast(null as geometry) AS geometry#261420, __MIMIR_ROWID#261601L, __MIMIR_ATTR_CAVEATTED__idx_261407#262066 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262072, __MIMIR_ATTR_CAVEATTED__file_261408#262067 AS __MIMIR_ATTR_CAVEATTED__file_261408#262073, __MIMIR_ATTR_CAVEATTED__lat_261409#262068 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262074, __MIMIR_ATTR_CAVEATTED__lon_261410#262069 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262075, false AS __MIMIR_ATTR_CAVEATTED__geometry_261420#262076, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261601#262070 AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261601#262077, __MIMIR_ROW_CAVEATTED#262065 AS __MIMIR_ROW_CAVEATTED#262071]
                          :        +- Project [idx#262327, file#262328, lat#262329, lon#262330, cast(hash(if (isnull(__MIMIR_ROWID_PARTITION_ID#261602L)) cast(__MIMIR_ROWID_INTERNAL_ID#261604L as bigint) else (cast(__MIMIR_ROWID_INTERNAL_ID#261604L as bigint) + cast(if (isnull(cast(__MIMIR_ROWID_PARTITION_ID#261602L as int))) null else FIRST_IDENTIFIER_FOR_PARTITION(knownnotnull(cast(__MIMIR_ROWID_PARTITION_ID#261602L as int))) as bigint)), idx#262327, file#262328, lat#262329, lon#262330, 42) as bigint) AS __MIMIR_ROWID#261601L, __MIMIR_ATTR_CAVEATTED__idx_261407#262059 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262066, __MIMIR_ATTR_CAVEATTED__file_261408#262060 AS __MIMIR_ATTR_CAVEATTED__file_261408#262067, __MIMIR_ATTR_CAVEATTED__lat_261409#262061 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262068, __MIMIR_ATTR_CAVEATTED__lon_261410#262062 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262069, (((((__MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261602#262063 OR if (isnull(__MIMIR_ROWID_PARTITION_ID#261602L)) __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_INTERNAL_ID_261604#262064 else (__MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_INTERNAL_ID_261604#262064 OR (__MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261602#262063 OR if (isnull(cast(__MIMIR_ROWID_PARTITION_ID#261602L as int))) false else __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261602#262063))) OR __MIMIR_ATTR_CAVEATTED__idx_261407#262059) OR __MIMIR_ATTR_CAVEATTED__file_261408#262060) OR __MIMIR_ATTR_CAVEATTED__lat_261409#262061) OR __MIMIR_ATTR_CAVEATTED__lon_261410#262062) AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261601#262070, __MIMIR_ROW_CAVEATTED#262058 AS __MIMIR_ROW_CAVEATTED#262065]
                          :           +- Project [idx#262327, file#262328, lat#262329, lon#262330, cast(SPARK_PARTITION_ID() as bigint) AS __MIMIR_ROWID_PARTITION_ID#261602L, cast(monotonically_increasing_id() as bigint) AS __MIMIR_ROWID_INTERNAL_ID#261604L, __MIMIR_ATTR_CAVEATTED__idx_261407#262054 AS __MIMIR_ATTR_CAVEATTED__idx_261407#262059, __MIMIR_ATTR_CAVEATTED__file_261408#262055 AS __MIMIR_ATTR_CAVEATTED__file_261408#262060, __MIMIR_ATTR_CAVEATTED__lat_261409#262056 AS __MIMIR_ATTR_CAVEATTED__lat_261409#262061, __MIMIR_ATTR_CAVEATTED__lon_261410#262057 AS __MIMIR_ATTR_CAVEATTED__lon_261410#262062, false AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261602#262063, false AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_INTERNAL_ID_261604#262064, __MIMIR_ROW_CAVEATTED#262053 AS __MIMIR_ROW_CAVEATTED#262058]
                          :              +- Project [idx#262327, file#262328, lat#262329, lon#262330, false AS __MIMIR_ATTR_CAVEATTED__idx_261407#262054, false AS __MIMIR_ATTR_CAVEATTED__file_261408#262055, false AS __MIMIR_ATTR_CAVEATTED__lat_261409#262056, false AS __MIMIR_ATTR_CAVEATTED__lon_261410#262057, false AS __MIMIR_ROW_CAVEATTED#262053]
                          :                 +- LocalRelation [idx#262327, file#262328, lat#262329, lon#262330]
                          +- Project [idx#262331, file#262332, lat#262333, lon#262334, geometry#261461, __MIMIR_ROWID#261778L AS RHS___MIMIR_ROWID#261860L, __MIMIR_ATTR_CAVEATTED__idx_261443#262118 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262125, __MIMIR_ATTR_CAVEATTED__file_261444#262119 AS __MIMIR_ATTR_CAVEATTED__file_261444#262126, __MIMIR_ATTR_CAVEATTED__lat_261445#262120 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262127, __MIMIR_ATTR_CAVEATTED__lon_261446#262121 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262128, __MIMIR_ATTR_CAVEATTED__geometry_261461#262122 AS __MIMIR_ATTR_CAVEATTED__geometry_261461#262129, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261778#262123 AS __MIMIR_ATTR_CAVEATTED__RHS___MIMIR_ROWID_261860#262130, __MIMIR_ROW_CAVEATTED#262117 AS __MIMIR_ROW_CAVEATTED#262124]
                             +- Project [idx#262331, file#262332, lat#262333, lon#262334, CASE WHEN (isnull(lon#262334) OR isnull(lat#262333)) THEN cast(null as geometry) ELSE st_point(cast(lon#262334 as decimal(24,20)), cast(lat#262333 as decimal(24,20))) END AS geometry#261461, __MIMIR_ROWID#261778L, __MIMIR_ATTR_CAVEATTED__idx_261443#262111 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262118, __MIMIR_ATTR_CAVEATTED__file_261444#262112 AS __MIMIR_ATTR_CAVEATTED__file_261444#262119, __MIMIR_ATTR_CAVEATTED__lat_261445#262113 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262120, __MIMIR_ATTR_CAVEATTED__lon_261446#262114 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262121, CASE WHEN (((__MIMIR_ATTR_CAVEATTED__lon_261446#262114 AND __MIMIR_ATTR_CAVEATTED__lat_261445#262113) OR (__MIMIR_ATTR_CAVEATTED__lon_261446#262114 AND NOT isnull(lat#262333))) OR (__MIMIR_ATTR_CAVEATTED__lat_261445#262113 AND NOT isnull(lon#262334))) THEN true WHEN (isnull(lon#262334) OR isnull(lat#262333)) THEN false ELSE (__MIMIR_ATTR_CAVEATTED__lon_261446#262114 OR __MIMIR_ATTR_CAVEATTED__lat_261445#262113) END AS __MIMIR_ATTR_CAVEATTED__geometry_261461#262122, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261778#262116 AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261778#262123, __MIMIR_ROW_CAVEATTED#262110 AS __MIMIR_ROW_CAVEATTED#262117]
                                +- Project [idx#262331, file#262332, lat#262333, lon#262334, cast(null as geometry) AS geometry#261453, __MIMIR_ROWID#261778L, __MIMIR_ATTR_CAVEATTED__idx_261443#262105 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262111, __MIMIR_ATTR_CAVEATTED__file_261444#262106 AS __MIMIR_ATTR_CAVEATTED__file_261444#262112, __MIMIR_ATTR_CAVEATTED__lat_261445#262107 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262113, __MIMIR_ATTR_CAVEATTED__lon_261446#262108 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262114, false AS __MIMIR_ATTR_CAVEATTED__geometry_261453#262115, __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261778#262109 AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261778#262116, __MIMIR_ROW_CAVEATTED#262104 AS __MIMIR_ROW_CAVEATTED#262110]
                                   +- Project [idx#262331, file#262332, lat#262333, lon#262334, cast(hash(if (isnull(__MIMIR_ROWID_PARTITION_ID#261779L)) cast(__MIMIR_ROWID_INTERNAL_ID#261781L as bigint) else (cast(__MIMIR_ROWID_INTERNAL_ID#261781L as bigint) + cast(if (isnull(cast(__MIMIR_ROWID_PARTITION_ID#261779L as int))) null else FIRST_IDENTIFIER_FOR_PARTITION(knownnotnull(cast(__MIMIR_ROWID_PARTITION_ID#261779L as int))) as bigint)), idx#262331, file#262332, lat#262333, lon#262334, 42) as bigint) AS __MIMIR_ROWID#261778L, __MIMIR_ATTR_CAVEATTED__idx_261443#262098 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262105, __MIMIR_ATTR_CAVEATTED__file_261444#262099 AS __MIMIR_ATTR_CAVEATTED__file_261444#262106, __MIMIR_ATTR_CAVEATTED__lat_261445#262100 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262107, __MIMIR_ATTR_CAVEATTED__lon_261446#262101 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262108, (((((__MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261779#262102 OR if (isnull(__MIMIR_ROWID_PARTITION_ID#261779L)) __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_INTERNAL_ID_261781#262103 else (__MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_INTERNAL_ID_261781#262103 OR (__MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261779#262102 OR if (isnull(cast(__MIMIR_ROWID_PARTITION_ID#261779L as int))) false else __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261779#262102))) OR __MIMIR_ATTR_CAVEATTED__idx_261443#262098) OR __MIMIR_ATTR_CAVEATTED__file_261444#262099) OR __MIMIR_ATTR_CAVEATTED__lat_261445#262100) OR __MIMIR_ATTR_CAVEATTED__lon_261446#262101) AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_261778#262109, __MIMIR_ROW_CAVEATTED#262097 AS __MIMIR_ROW_CAVEATTED#262104]
                                      +- Project [idx#262331, file#262332, lat#262333, lon#262334, cast(SPARK_PARTITION_ID() as bigint) AS __MIMIR_ROWID_PARTITION_ID#261779L, cast(monotonically_increasing_id() as bigint) AS __MIMIR_ROWID_INTERNAL_ID#261781L, __MIMIR_ATTR_CAVEATTED__idx_261443#262093 AS __MIMIR_ATTR_CAVEATTED__idx_261443#262098, __MIMIR_ATTR_CAVEATTED__file_261444#262094 AS __MIMIR_ATTR_CAVEATTED__file_261444#262099, __MIMIR_ATTR_CAVEATTED__lat_261445#262095 AS __MIMIR_ATTR_CAVEATTED__lat_261445#262100, __MIMIR_ATTR_CAVEATTED__lon_261446#262096 AS __MIMIR_ATTR_CAVEATTED__lon_261446#262101, false AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_PARTITION_ID_261779#262102, false AS __MIMIR_ATTR_CAVEATTED____MIMIR_ROWID_INTERNAL_ID_261781#262103, __MIMIR_ROW_CAVEATTED#262092 AS __MIMIR_ROW_CAVEATTED#262097]
                                         +- Project [idx#262331, file#262332, lat#262333, lon#262334, false AS __MIMIR_ATTR_CAVEATTED__idx_261443#262093, false AS __MIMIR_ATTR_CAVEATTED__file_261444#262094, false AS __MIMIR_ATTR_CAVEATTED__lat_261445#262095, false AS __MIMIR_ATTR_CAVEATTED__lon_261446#262096, false AS __MIMIR_ROW_CAVEATTED#262092]
                                            +- LocalRelation [idx#262331, file#262332, lat#262333, lon#262334]
@okennedy okennedy mentioned this issue Oct 9, 2023
Closed
@okennedy
Copy link
Member Author

okennedy commented Oct 9, 2023

I think the problem is at:
https://github.com/UBOdin/mimir-caveats/blob/master/src/main/scala/org/mimirdb/caveats/annotate/CaveatExistsInPlan.scala#L465

Specifically, annotate is getting a plan that references a different set of attributes... I'm reasonably certain that it should be taking ret and not plan, but making that change breaks unit tests, so I need to dig further.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
None yet
Projects
None yet
Milestone
No milestone
Development

No branches or pull requests
1 participant
@okennedy