more robust colum names

main.py

@@ -75,16 +75,16 @@ async def get_first_config_and_split_name(hub_id: str):
         return data["splits"][0]["config"], data["splits"][0]["split"]
     except Exception as e:
         logger.error(f"Failed to get splits for {hub_id}: {e}")
-        return None
+        return (None, None)  # Return a tuple of None values
 
 
 async def get_dataset_info(hub_id: str, config: str | None = None):
     if config is None:
-        config = get_first_config_and_split_name(hub_id)
-        if config is None:
+        config_tuple, _ = await get_first_config_and_split_name(hub_id)
+        if config_tuple is None:
             return None
         else:
-            config = config[0]
+            config = config_tuple
     resp = await async_client.get(
         f"{BASE_DATASETS_SERVER_URL}/info?dataset={hub_id}&config={config}"
     )
@@ -229,18 +229,10 @@ def predict_rows(
     return default_data
 
 
-# @app.get("/", response_class=HTMLResponse)
-# async def read_index():
-#     html_content = Path("index.html").read_text()
-#     return HTMLResponse(content=html_content)
-
-
 @app.get("/", include_in_schema=False)
 def root():
     return RedirectResponse(url="/docs")
 
-    # item_id: Annotated[int, Path(title="The ID of the item to get", ge=1)], q: str
-
 
 @app.get("/predict_dataset_language/{hub_id:path}")
 @cache(ttl=timedelta(minutes=10))
@@ -257,31 +249,66 @@ async def predict_language(
     is_valid = datasets_server_valid_rows(hub_id)
     if not is_valid:
         logger.error(f"Dataset {hub_id} is not accessible via the datasets server.")
+        return None  # Return early if dataset is not valid
+        
     if not config and not split:
-        config, split = await get_first_config_and_split_name(hub_id)
-    if not config:
-        config, _ = await get_first_config_and_split_name(hub_id)
-    if not split:
-        _, split = await get_first_config_and_split_name(hub_id)
+        config_tuple, split_tuple = await get_first_config_and_split_name(hub_id)
+        if config_tuple is None:
+            logger.error(f"Could not retrieve configuration for dataset {hub_id}")
+            return None
+        config, split = config_tuple, split_tuple
+    elif not config:
+        config_tuple, _ = await get_first_config_and_split_name(hub_id)
+        if config_tuple is None:
+            logger.error(f"Could not retrieve configuration for dataset {hub_id}")
+            return None
+        config = config_tuple
+    elif not split:
+        _, split_tuple = await get_first_config_and_split_name(hub_id)
+        if split_tuple is None:
+            logger.error(f"Could not retrieve split for dataset {hub_id}")
+            return None
+        split = split_tuple
+        
     info = await get_dataset_info(hub_id, config)
     if info is None:
         logger.error(f"Dataset {hub_id} is not accessible via the datasets server.")
         return None
+        
     if dataset_info := info.get("dataset_info"):
         total_rows_for_split = dataset_info.get("splits").get(split).get("num_examples")
         features = dataset_info.get("features")
+        
+        # Get original column names
         column_names = set(features.keys())
         logger.info(f"Column names: {column_names}")
-        if not set(column_names).intersection(TARGET_COLUMN_NAMES):
+        
+        # Create a mapping of lowercase column names to their original casing
+        lowercase_to_original = {col.lower(): col for col in column_names}
+        
+        # Check intersection with lowercase versions
+        lowercase_column_names = set(lowercase_to_original.keys())
+        lowercase_target_columns = {col.lower() for col in TARGET_COLUMN_NAMES}
+        
+        if not lowercase_column_names.intersection(lowercase_target_columns):
             logger.error(
-                f"Dataset {hub_id} {column_names} is not in any of the target columns {TARGET_COLUMN_NAMES}"
+                f"Dataset {hub_id} {column_names} does not contain any of the target columns {TARGET_COLUMN_NAMES}"
             )
             return None
+            
+        # Find target column with case-insensitive matching
+        target_column = None
         for column in TARGET_COLUMN_NAMES:
-            if column in column_names:
-                target_column = column
+            if column.lower() in lowercase_column_names:
+                # Use the original casing from the dataset
+                target_column = lowercase_to_original[column.lower()]
                 logger.info(f"Using column {target_column} for language detection")
                 break
+                
+        if target_column is None:
+            logger.error(f"Could not find a suitable column for language detection")
+            return None
+            
         random_rows = await get_random_rows(
             hub_id,
             total_rows_for_split,
@@ -290,6 +317,7 @@ async def predict_language(
             config,
             split,
         )
+        
         logger.info(f"Predicting language for {len(random_rows)} rows")
         predictions = predict_rows(
             random_rows,
@@ -300,3 +328,7 @@ async def predict_language(
         predictions["config"] = config
         predictions["split"] = split
         return predictions
+        
+    else:
+        logger.error(f"No dataset_info available for {hub_id}")
+        return None