{ "schemaVersion": "1.0", "item": { "slug": "df-merger", "name": "Df Merger", "source": "tencent", "type": "skill", "category": "AI 智能", "sourceUrl": "https://clawhub.ai/datadrivenconstruction/df-merger", "canonicalUrl": "https://clawhub.ai/datadrivenconstruction/df-merger", "targetPlatform": "OpenClaw" }, "install": { "downloadMode": "redirect", "downloadUrl": "/downloads/df-merger", "sourceDownloadUrl": "https://wry-manatee-359.convex.site/api/v1/download?slug=df-merger", "sourcePlatform": "tencent", "targetPlatform": "OpenClaw", "installMethod": "Manual import", "extraction": "Extract archive", "prerequisites": [ "OpenClaw" ], "packageFormat": "ZIP package", "includedAssets": [ "claw.json", "instructions.md", "SKILL.md" ], "primaryDoc": "SKILL.md", "quickSetup": [ "Download the package from Yavira.", "Extract the archive and review SKILL.md first.", "Import or place the package into your OpenClaw setup." ], "agentAssist": { "summary": "Hand the extracted package to your coding agent with a concrete install brief instead of figuring it out manually.", "steps": [ "Download the package from Yavira.", "Extract it into a folder your agent can access.", "Paste one of the prompts below and point your agent at the extracted folder." ], "prompts": [ { "label": "New install", "body": "I downloaded a skill package from Yavira. Read SKILL.md from the extracted folder and install it by following the included instructions. Tell me what you changed and call out any manual steps you could not complete." }, { "label": "Upgrade existing", "body": "I downloaded an updated skill package from Yavira. Read SKILL.md from the extracted folder, compare it with my current installation, and upgrade it while preserving any custom configuration unless the package docs explicitly say otherwise. Summarize what changed and any follow-up checks I should run." } ] }, "sourceHealth": { "source": "tencent", "status": "healthy", "reason": "direct_download_ok", "recommendedAction": "download", "checkedAt": "2026-04-30T16:55:25.780Z", "expiresAt": "2026-05-07T16:55:25.780Z", "httpStatus": 200, "finalUrl": "https://wry-manatee-359.convex.site/api/v1/download?slug=network", "contentType": "application/zip", "probeMethod": "head", "details": { "probeUrl": "https://wry-manatee-359.convex.site/api/v1/download?slug=network", "contentDisposition": "attachment; filename=\"network-1.0.0.zip\"", "redirectLocation": null, "bodySnippet": null }, "scope": "source", "summary": "Source download looks usable.", "detail": "Yavira can redirect you to the upstream package for this source.", "primaryActionLabel": "Download for OpenClaw", "primaryActionHref": "/downloads/df-merger" }, "validation": { "installChecklist": [ "Use the Yavira download entry.", "Review SKILL.md after the package is downloaded.", "Confirm the extracted package contains the expected setup assets." ], "postInstallChecks": [ "Confirm the extracted package includes the expected docs or setup files.", "Validate the skill or prompts are available in your target agent workspace.", "Capture any manual follow-up steps the agent could not complete." ] }, "downloadPageUrl": "https://openagent3.xyz/downloads/df-merger", "agentPageUrl": "https://openagent3.xyz/skills/df-merger/agent", "manifestUrl": "https://openagent3.xyz/skills/df-merger/agent.json", "briefUrl": "https://openagent3.xyz/skills/df-merger/agent.md" }, "agentAssist": { "summary": "Hand the extracted package to your coding agent with a concrete install brief instead of figuring it out manually.", "steps": [ "Download the package from Yavira.", "Extract it into a folder your agent can access.", "Paste one of the prompts below and point your agent at the extracted folder." ], "prompts": [ { "label": "New install", "body": "I downloaded a skill package from Yavira. Read SKILL.md from the extracted folder and install it by following the included instructions. Tell me what you changed and call out any manual steps you could not complete." }, { "label": "Upgrade existing", "body": "I downloaded an updated skill package from Yavira. Read SKILL.md from the extracted folder, compare it with my current installation, and upgrade it while preserving any custom configuration unless the package docs explicitly say otherwise. Summarize what changed and any follow-up checks I should run." } ] }, "documentation": { "source": "clawhub", "primaryDoc": "SKILL.md", "sections": [ { "title": "Overview", "body": "Construction projects combine data from BIM, schedules, costs, and sensors. This skill merges DataFrames from disparate sources with intelligent key matching and schema reconciliation." }, { "title": "Python Implementation", "body": "import pandas as pd\nimport numpy as np\nfrom typing import Dict, Any, List, Optional, Tuple\nfrom dataclasses import dataclass\nfrom enum import Enum\nfrom difflib import SequenceMatcher\n\n\nclass MergeStrategy(Enum):\n \"\"\"DataFrame merge strategies.\"\"\"\n INNER = \"inner\" # Only matching rows\n LEFT = \"left\" # All left, matching right\n RIGHT = \"right\" # Matching left, all right\n OUTER = \"outer\" # All rows from both\n CROSS = \"cross\" # Cartesian product\n\n\n@dataclass\nclass MergeResult:\n \"\"\"Result of merge operation.\"\"\"\n merged_df: pd.DataFrame\n matched_rows: int\n left_only: int\n right_only: int\n merge_quality: float # 0-1 score\n\n\nclass ConstructionDFMerger:\n \"\"\"Merge DataFrames from construction sources.\"\"\"\n\n # Common construction column name mappings\n COLUMN_MAPPINGS = {\n 'element_id': ['elementid', 'elem_id', 'id', 'guid', 'globalid'],\n 'type_name': ['typename', 'type', 'element_type', 'category'],\n 'level': ['level', 'floor', 'storey', 'building_storey'],\n 'material': ['material', 'mat', 'material_name'],\n 'volume': ['volume', 'vol', 'volume_m3', 'qty_volume'],\n 'area': ['area', 'surface_area', 'qty_area', 'area_m2'],\n 'cost': ['cost', 'price', 'total_cost', 'amount'],\n 'task_id': ['task_id', 'activity_id', 'wbs', 'activity'],\n 'start_date': ['start', 'start_date', 'planned_start', 'begin'],\n 'end_date': ['end', 'end_date', 'planned_finish', 'finish']\n }\n\n def __init__(self):\n self.column_cache: Dict[str, str] = {}\n\n def find_common_key(self, df1: pd.DataFrame,\n df2: pd.DataFrame) -> Optional[str]:\n \"\"\"Find common key column between DataFrames.\"\"\"\n\n # Check exact matches first\n common = set(df1.columns) & set(df2.columns)\n if common:\n # Prefer ID-like columns\n for col in common:\n if 'id' in col.lower() or 'code' in col.lower():\n return col\n return list(common)[0]\n\n # Try semantic matching\n for col1 in df1.columns:\n for col2 in df2.columns:\n if self._columns_match(col1, col2):\n return col1\n\n return None\n\n def _columns_match(self, col1: str, col2: str) -> bool:\n \"\"\"Check if column names are semantically similar.\"\"\"\n col1_lower = col1.lower().replace('_', '').replace('-', '')\n col2_lower = col2.lower().replace('_', '').replace('-', '')\n\n # Exact match after normalization\n if col1_lower == col2_lower:\n return True\n\n # Check against mappings\n for standard, variants in self.COLUMN_MAPPINGS.items():\n if col1_lower in variants and col2_lower in variants:\n return True\n\n # Similarity check\n similarity = SequenceMatcher(None, col1_lower, col2_lower).ratio()\n return similarity > 0.8\n\n def harmonize_columns(self, df: pd.DataFrame) -> pd.DataFrame:\n \"\"\"Standardize column names.\"\"\"\n df = df.copy()\n rename_map = {}\n\n for col in df.columns:\n col_lower = col.lower().replace('_', '').replace('-', '')\n\n for standard, variants in self.COLUMN_MAPPINGS.items():\n if col_lower in variants:\n rename_map[col] = standard\n break\n\n return df.rename(columns=rename_map)\n\n def merge(self, left: pd.DataFrame,\n right: pd.DataFrame,\n on: Optional[str] = None,\n left_on: Optional[str] = None,\n right_on: Optional[str] = None,\n how: MergeStrategy = MergeStrategy.LEFT,\n harmonize: bool = True) -> MergeResult:\n \"\"\"Merge two DataFrames.\"\"\"\n\n if harmonize:\n left = self.harmonize_columns(left)\n right = self.harmonize_columns(right)\n\n # Determine merge keys\n if on is None and left_on is None and right_on is None:\n common_key = self.find_common_key(left, right)\n if common_key is None:\n raise ValueError(\"No common key found. Specify merge key manually.\")\n on = common_key\n\n # Perform merge\n merged = pd.merge(\n left, right,\n on=on,\n left_on=left_on,\n right_on=right_on,\n how=how.value,\n indicator=True,\n suffixes=('_left', '_right')\n )\n\n # Calculate statistics\n matched = len(merged[merged['_merge'] == 'both'])\n left_only = len(merged[merged['_merge'] == 'left_only'])\n right_only = len(merged[merged['_merge'] == 'right_only'])\n\n # Quality score\n total = len(left) + len(right)\n quality = (matched * 2) / total if total > 0 else 0\n\n # Clean up\n merged = merged.drop('_merge', axis=1)\n\n return MergeResult(\n merged_df=merged,\n matched_rows=matched,\n left_only=left_only,\n right_only=right_only,\n merge_quality=round(quality, 2)\n )\n\n def merge_multiple(self, dfs: List[pd.DataFrame],\n on: Optional[str] = None,\n how: MergeStrategy = MergeStrategy.OUTER) -> pd.DataFrame:\n \"\"\"Merge multiple DataFrames sequentially.\"\"\"\n\n if not dfs:\n return pd.DataFrame()\n\n result = dfs[0].copy()\n\n for i, df in enumerate(dfs[1:], 1):\n result_obj = self.merge(result, df, on=on, how=how)\n result = result_obj.merged_df\n\n return result\n\n def fuzzy_merge(self, left: pd.DataFrame,\n right: pd.DataFrame,\n left_on: str,\n right_on: str,\n threshold: float = 0.8) -> pd.DataFrame:\n \"\"\"Merge using fuzzy string matching.\"\"\"\n\n matches = []\n\n left_values = left[left_on].dropna().unique()\n right_values = right[right_on].dropna().unique()\n\n for lval in left_values:\n best_match = None\n best_score = 0\n\n for rval in right_values:\n score = SequenceMatcher(None, str(lval).lower(),\n str(rval).lower()).ratio()\n if score > best_score and score >= threshold:\n best_score = score\n best_match = rval\n\n if best_match:\n matches.append({\n 'left_key': lval,\n 'right_key': best_match,\n 'match_score': best_score\n })\n\n match_df = pd.DataFrame(matches)\n\n # Join using match mapping\n left_with_key = left.merge(match_df, left_on=left_on, right_on='left_key', how='left')\n result = left_with_key.merge(right, left_on='right_key', right_on=right_on, how='left')\n\n return result\n\n\nclass BIMScheduleMerger(ConstructionDFMerger):\n \"\"\"Specialized merger for BIM and schedule data.\"\"\"\n\n def merge_bim_schedule(self, bim_df: pd.DataFrame,\n schedule_df: pd.DataFrame,\n bim_type_col: str = 'Type Name',\n schedule_wbs_col: str = 'WBS') -> pd.DataFrame:\n \"\"\"Merge BIM elements with schedule activities.\"\"\"\n\n # This typically requires a mapping table\n # For now, use fuzzy matching on descriptions\n\n bim_df = self.harmonize_columns(bim_df)\n schedule_df = self.harmonize_columns(schedule_df)\n\n # Try to match type names to WBS descriptions\n result = self.fuzzy_merge(\n bim_df, schedule_df,\n left_on=bim_type_col,\n right_on=schedule_wbs_col,\n threshold=0.6\n )\n\n return result\n\n\nclass CostQTOMerger(ConstructionDFMerger):\n \"\"\"Merge cost data with quantity takeoffs.\"\"\"\n\n def merge_cost_qto(self, cost_df: pd.DataFrame,\n qto_df: pd.DataFrame) -> pd.DataFrame:\n \"\"\"Merge cost rates with QTO quantities.\"\"\"\n\n cost_df = self.harmonize_columns(cost_df)\n qto_df = self.harmonize_columns(qto_df)\n\n # Try common merge keys\n for key in ['work_item_code', 'type_name', 'material', 'element_id']:\n if key in cost_df.columns and key in qto_df.columns:\n result = self.merge(cost_df, qto_df, on=key)\n\n # Calculate extended costs\n result.merged_df['extended_cost'] = (\n result.merged_df.get('quantity', 0) *\n result.merged_df.get('unit_price', 0)\n )\n\n return result.merged_df\n\n # Fallback to fuzzy merge\n return self.fuzzy_merge(\n qto_df, cost_df,\n left_on='type_name' if 'type_name' in qto_df.columns else qto_df.columns[0],\n right_on='description' if 'description' in cost_df.columns else cost_df.columns[0]\n )" }, { "title": "Quick Start", "body": "merger = ConstructionDFMerger()\n\n# Merge two DataFrames\nresult = merger.merge(bim_df, schedule_df)\nprint(f\"Matched: {result.matched_rows}, Quality: {result.merge_quality}\")\n\n# Access merged data\nmerged = result.merged_df" }, { "title": "1. BIM + Schedule Integration", "body": "bim_schedule = BIMScheduleMerger()\nintegrated = bim_schedule.merge_bim_schedule(bim_elements, schedule_activities)" }, { "title": "2. Cost + QTO", "body": "cost_merger = CostQTOMerger()\npriced_qto = cost_merger.merge_cost_qto(cost_database, quantities)\nprint(f\"Total: ${priced_qto['extended_cost'].sum():,.2f}\")" }, { "title": "3. Multiple Sources", "body": "all_data = merger.merge_multiple(\n [bim_df, schedule_df, cost_df, resource_df],\n on='element_id'\n)" }, { "title": "Resources", "body": "DDC Book: Chapter 2.3 - Pandas DataFrame" } ], "body": "DataFrame Merger for Construction Data\nOverview\n\nConstruction projects combine data from BIM, schedules, costs, and sensors. This skill merges DataFrames from disparate sources with intelligent key matching and schema reconciliation.\n\nPython Implementation\nimport pandas as pd\nimport numpy as np\nfrom typing import Dict, Any, List, Optional, Tuple\nfrom dataclasses import dataclass\nfrom enum import Enum\nfrom difflib import SequenceMatcher\n\n\nclass MergeStrategy(Enum):\n \"\"\"DataFrame merge strategies.\"\"\"\n INNER = \"inner\" # Only matching rows\n LEFT = \"left\" # All left, matching right\n RIGHT = \"right\" # Matching left, all right\n OUTER = \"outer\" # All rows from both\n CROSS = \"cross\" # Cartesian product\n\n\n@dataclass\nclass MergeResult:\n \"\"\"Result of merge operation.\"\"\"\n merged_df: pd.DataFrame\n matched_rows: int\n left_only: int\n right_only: int\n merge_quality: float # 0-1 score\n\n\nclass ConstructionDFMerger:\n \"\"\"Merge DataFrames from construction sources.\"\"\"\n\n # Common construction column name mappings\n COLUMN_MAPPINGS = {\n 'element_id': ['elementid', 'elem_id', 'id', 'guid', 'globalid'],\n 'type_name': ['typename', 'type', 'element_type', 'category'],\n 'level': ['level', 'floor', 'storey', 'building_storey'],\n 'material': ['material', 'mat', 'material_name'],\n 'volume': ['volume', 'vol', 'volume_m3', 'qty_volume'],\n 'area': ['area', 'surface_area', 'qty_area', 'area_m2'],\n 'cost': ['cost', 'price', 'total_cost', 'amount'],\n 'task_id': ['task_id', 'activity_id', 'wbs', 'activity'],\n 'start_date': ['start', 'start_date', 'planned_start', 'begin'],\n 'end_date': ['end', 'end_date', 'planned_finish', 'finish']\n }\n\n def __init__(self):\n self.column_cache: Dict[str, str] = {}\n\n def find_common_key(self, df1: pd.DataFrame,\n df2: pd.DataFrame) -> Optional[str]:\n \"\"\"Find common key column between DataFrames.\"\"\"\n\n # Check exact matches first\n common = set(df1.columns) & set(df2.columns)\n if common:\n # Prefer ID-like columns\n for col in common:\n if 'id' in col.lower() or 'code' in col.lower():\n return col\n return list(common)[0]\n\n # Try semantic matching\n for col1 in df1.columns:\n for col2 in df2.columns:\n if self._columns_match(col1, col2):\n return col1\n\n return None\n\n def _columns_match(self, col1: str, col2: str) -> bool:\n \"\"\"Check if column names are semantically similar.\"\"\"\n col1_lower = col1.lower().replace('_', '').replace('-', '')\n col2_lower = col2.lower().replace('_', '').replace('-', '')\n\n # Exact match after normalization\n if col1_lower == col2_lower:\n return True\n\n # Check against mappings\n for standard, variants in self.COLUMN_MAPPINGS.items():\n if col1_lower in variants and col2_lower in variants:\n return True\n\n # Similarity check\n similarity = SequenceMatcher(None, col1_lower, col2_lower).ratio()\n return similarity > 0.8\n\n def harmonize_columns(self, df: pd.DataFrame) -> pd.DataFrame:\n \"\"\"Standardize column names.\"\"\"\n df = df.copy()\n rename_map = {}\n\n for col in df.columns:\n col_lower = col.lower().replace('_', '').replace('-', '')\n\n for standard, variants in self.COLUMN_MAPPINGS.items():\n if col_lower in variants:\n rename_map[col] = standard\n break\n\n return df.rename(columns=rename_map)\n\n def merge(self, left: pd.DataFrame,\n right: pd.DataFrame,\n on: Optional[str] = None,\n left_on: Optional[str] = None,\n right_on: Optional[str] = None,\n how: MergeStrategy = MergeStrategy.LEFT,\n harmonize: bool = True) -> MergeResult:\n \"\"\"Merge two DataFrames.\"\"\"\n\n if harmonize:\n left = self.harmonize_columns(left)\n right = self.harmonize_columns(right)\n\n # Determine merge keys\n if on is None and left_on is None and right_on is None:\n common_key = self.find_common_key(left, right)\n if common_key is None:\n raise ValueError(\"No common key found. Specify merge key manually.\")\n on = common_key\n\n # Perform merge\n merged = pd.merge(\n left, right,\n on=on,\n left_on=left_on,\n right_on=right_on,\n how=how.value,\n indicator=True,\n suffixes=('_left', '_right')\n )\n\n # Calculate statistics\n matched = len(merged[merged['_merge'] == 'both'])\n left_only = len(merged[merged['_merge'] == 'left_only'])\n right_only = len(merged[merged['_merge'] == 'right_only'])\n\n # Quality score\n total = len(left) + len(right)\n quality = (matched * 2) / total if total > 0 else 0\n\n # Clean up\n merged = merged.drop('_merge', axis=1)\n\n return MergeResult(\n merged_df=merged,\n matched_rows=matched,\n left_only=left_only,\n right_only=right_only,\n merge_quality=round(quality, 2)\n )\n\n def merge_multiple(self, dfs: List[pd.DataFrame],\n on: Optional[str] = None,\n how: MergeStrategy = MergeStrategy.OUTER) -> pd.DataFrame:\n \"\"\"Merge multiple DataFrames sequentially.\"\"\"\n\n if not dfs:\n return pd.DataFrame()\n\n result = dfs[0].copy()\n\n for i, df in enumerate(dfs[1:], 1):\n result_obj = self.merge(result, df, on=on, how=how)\n result = result_obj.merged_df\n\n return result\n\n def fuzzy_merge(self, left: pd.DataFrame,\n right: pd.DataFrame,\n left_on: str,\n right_on: str,\n threshold: float = 0.8) -> pd.DataFrame:\n \"\"\"Merge using fuzzy string matching.\"\"\"\n\n matches = []\n\n left_values = left[left_on].dropna().unique()\n right_values = right[right_on].dropna().unique()\n\n for lval in left_values:\n best_match = None\n best_score = 0\n\n for rval in right_values:\n score = SequenceMatcher(None, str(lval).lower(),\n str(rval).lower()).ratio()\n if score > best_score and score >= threshold:\n best_score = score\n best_match = rval\n\n if best_match:\n matches.append({\n 'left_key': lval,\n 'right_key': best_match,\n 'match_score': best_score\n })\n\n match_df = pd.DataFrame(matches)\n\n # Join using match mapping\n left_with_key = left.merge(match_df, left_on=left_on, right_on='left_key', how='left')\n result = left_with_key.merge(right, left_on='right_key', right_on=right_on, how='left')\n\n return result\n\n\nclass BIMScheduleMerger(ConstructionDFMerger):\n \"\"\"Specialized merger for BIM and schedule data.\"\"\"\n\n def merge_bim_schedule(self, bim_df: pd.DataFrame,\n schedule_df: pd.DataFrame,\n bim_type_col: str = 'Type Name',\n schedule_wbs_col: str = 'WBS') -> pd.DataFrame:\n \"\"\"Merge BIM elements with schedule activities.\"\"\"\n\n # This typically requires a mapping table\n # For now, use fuzzy matching on descriptions\n\n bim_df = self.harmonize_columns(bim_df)\n schedule_df = self.harmonize_columns(schedule_df)\n\n # Try to match type names to WBS descriptions\n result = self.fuzzy_merge(\n bim_df, schedule_df,\n left_on=bim_type_col,\n right_on=schedule_wbs_col,\n threshold=0.6\n )\n\n return result\n\n\nclass CostQTOMerger(ConstructionDFMerger):\n \"\"\"Merge cost data with quantity takeoffs.\"\"\"\n\n def merge_cost_qto(self, cost_df: pd.DataFrame,\n qto_df: pd.DataFrame) -> pd.DataFrame:\n \"\"\"Merge cost rates with QTO quantities.\"\"\"\n\n cost_df = self.harmonize_columns(cost_df)\n qto_df = self.harmonize_columns(qto_df)\n\n # Try common merge keys\n for key in ['work_item_code', 'type_name', 'material', 'element_id']:\n if key in cost_df.columns and key in qto_df.columns:\n result = self.merge(cost_df, qto_df, on=key)\n\n # Calculate extended costs\n result.merged_df['extended_cost'] = (\n result.merged_df.get('quantity', 0) *\n result.merged_df.get('unit_price', 0)\n )\n\n return result.merged_df\n\n # Fallback to fuzzy merge\n return self.fuzzy_merge(\n qto_df, cost_df,\n left_on='type_name' if 'type_name' in qto_df.columns else qto_df.columns[0],\n right_on='description' if 'description' in cost_df.columns else cost_df.columns[0]\n )\n\nQuick Start\nmerger = ConstructionDFMerger()\n\n# Merge two DataFrames\nresult = merger.merge(bim_df, schedule_df)\nprint(f\"Matched: {result.matched_rows}, Quality: {result.merge_quality}\")\n\n# Access merged data\nmerged = result.merged_df\n\nCommon Use Cases\n1. BIM + Schedule Integration\nbim_schedule = BIMScheduleMerger()\nintegrated = bim_schedule.merge_bim_schedule(bim_elements, schedule_activities)\n\n2. Cost + QTO\ncost_merger = CostQTOMerger()\npriced_qto = cost_merger.merge_cost_qto(cost_database, quantities)\nprint(f\"Total: ${priced_qto['extended_cost'].sum():,.2f}\")\n\n3. Multiple Sources\nall_data = merger.merge_multiple(\n [bim_df, schedule_df, cost_df, resource_df],\n on='element_id'\n)\n\nResources\nDDC Book: Chapter 2.3 - Pandas DataFrame" }, "trust": { "sourceLabel": "tencent", "provenanceUrl": "https://clawhub.ai/datadrivenconstruction/df-merger", "publisherUrl": "https://clawhub.ai/datadrivenconstruction/df-merger", "owner": "datadrivenconstruction", "version": "2.1.0", "license": null, "verificationStatus": "Indexed source record" }, "links": { "detailUrl": "https://openagent3.xyz/skills/df-merger", "downloadUrl": "https://openagent3.xyz/downloads/df-merger", "agentUrl": "https://openagent3.xyz/skills/df-merger/agent", "manifestUrl": "https://openagent3.xyz/skills/df-merger/agent.json", "briefUrl": "https://openagent3.xyz/skills/df-merger/agent.md" } }