Add core-1 dataset

.gitignore

@@ -103,5 +103,5 @@ venv.bak/
 # mypy
 .mypy_cache/
 
-data/
+data/*.parquet
 script.py

components.py

@@ -6,9 +6,11 @@ import param
 from holoviews.operation.datashader import dynspread, rasterize
 
 from utils import (
+    DATASET_COLUMNS,
     DATASETS,
     DATASHADER_LOGO,
     DATASHADER_URL,
+    DEFAULT_DATASET,
     DESCRIPTION,
     ESA_EASTING,
     ESA_NORTHING,
@@ -16,7 +18,6 @@ from utils import (
     MAJOR_TOM_LYRICS,
     MAJOR_TOM_PICTURE,
     MAJOR_TOM_REF_URL,
-    META_DATA_COLUMNS,
     PANEL_LOGO,
     PANEL_URL,
     get_closest_rows,
@@ -26,9 +27,16 @@ from utils import (
 
 
 class DatasetInput(pn.viewable.Viewer):
-    value = param.Selector(objects=DATASETS, allow_None=False, label="Dataset")
+    value = param.Selector(
+        default=DEFAULT_DATASET,
+        objects=DATASETS,
+        allow_None=False,
+        label="Dataset",
+        doc="""The name of the dataset""",
+    )
 
-    data = param.DataFrame(allow_None=False)
+    data = param.DataFrame(allow_None=False, doc="""The metadata dataset""")
+    columns = param.Dict(allow_None=False, doc="""The columns of the dataset""")
 
     def __panel__(self):
         return pn.widgets.RadioButtonGroup.from_param(
@@ -37,7 +45,9 @@ class DatasetInput(pn.viewable.Viewer):
 
     @pn.depends("value", watch=True, on_init=True)
     def _update_data(self):
-        self.data = pn.cache(get_meta_data)(dataset=self.value)
+        columns = DATASET_COLUMNS[self.value]
+        data = pn.cache(get_meta_data)(dataset=self.value)
+        self.param.update(columns=columns, data=data)
 
 
 class MapInput(pn.viewable.Viewer):
@@ -129,17 +139,25 @@ class MapInput(pn.viewable.Viewer):
 
 
 class ImageInput(pn.viewable.Viewer):
-    data = param.DataFrame(allow_refs=True, allow_None=False)
+    data = param.DataFrame(
+        allow_refs=True, allow_None=False, doc="""The metadata selected"""
+    )
+    columns = param.Dict(
+        allow_refs=True, allow_None=False, doc="""The list of columns of the dataset"""
+    )
     column_name = param.Selector(
-        default="Thumbnail", objects=list(META_DATA_COLUMNS), label="Image Type"
+        label="Image Type",
+        allow_None=False,
+        doc="""The name of the image type to view""",
     )
+
     updating = param.Boolean()
 
     meta_data = param.DataFrame()
     image = param.Parameter()
     plot = param.Parameter()
 
-    _timestamp = param.Selector(label="Timestamp", objects=[None])
+    _timestamp = param.Selector(label="Timestamp", objects=[None], doc="""The timestamp of the sample to view""")
 
     def __panel__(self):
         return pn.Column(
@@ -182,23 +200,30 @@ class ImageInput(pn.viewable.Viewer):
         if self.data.empty:
             default_value = None
             options = [None]
-            print("empty options")
         else:
             options = sorted(self.data["timestamp"].unique())
             default_value = options[0]
-            print("options", options)
 
         self.param._timestamp.objects = options
         if not self._timestamp in options:
             self._timestamp = default_value
 
+    @pn.depends("columns", watch=True, on_init=True)
+    def _update_column_names(self):
+        options = sorted(self.columns)
+        default_value = "Thumbnail"
+
+        self.param.column_name.objects = options
+        if not self.column_name in options:
+            self.column_name = default_value
+
     @property
     def column(self):
-        return META_DATA_COLUMNS[self.column_name]
+        return self.columns[self.column_name]
 
     @pn.depends("_timestamp", "column_name", watch=True, on_init=True)
     def _update_plot(self):
-        if self.data.empty or not self._timestamp:
+        if self.data.empty or not self._timestamp or not self.column_name:
             self.meta_data = self.data.T
             self.image = None
             self.plot = hv.RGB(np.array([]))
@@ -309,7 +334,9 @@ class App(param.Parameterized):
     def _create_main_content(self):
         dataset = DatasetInput()
         map_input = MapInput(data=dataset.param.data)
-        image_input = ImageInput(data=map_input.param.data_selected)
+        image_input = ImageInput(
+            data=map_input.param.data_selected, columns=dataset.param.columns
+        )
 
         return pn.Column(dataset, map_input), image_input
 

download_datasets.py

@@ -2,6 +2,7 @@ from utils import DATASETS, get_meta_data
 
 for dataset in DATASETS:
     print(f"downloading {dataset}")
-    get_meta_data(dataset=dataset)
+    data = get_meta_data(dataset=dataset)
+
 
 print("finished downloading datasets")

utils.py

@@ -1,14 +1,18 @@
+import logging
 from io import BytesIO
 from pathlib import Path
 
 import holoviews as hv
 import numpy as np
 import pandas as pd
+import panel as pn
 import pyarrow.parquet as pq
 from fsspec.parquet import open_parquet_file
 from holoviews import opts
 from PIL import Image
 
+FORMAT = "%(asctime)s | %(levelname)s | %(name)s | %(message)s"
+
 MAJOR_TOM_LOGO = "assets/major-tom-esa-logo.png"
 MAJOR_TOM_PICTURE = (
     "https://upload.wikimedia.org/wikipedia/en/6/6d/Major_tom_space_oddity_video.JPG"
@@ -20,10 +24,16 @@ PANEL_URL = "https://panel.holoviz.org"
 DATASHADER_LOGO = "https://datashader.org/_static/logo_horizontal.svg"
 DATASHADER_URL = "https://datashader.org/"
 REPOSITORY = "Major-TOM"
-DATASETS = ["Core-S2L2A", "Core-S2L1C"]
+DEFAULT_DATASET = "Core-S2L2A"
+DATASETS = ["Core-S1RTC", "Core-S2L2A", "Core-S2L1C"]
 ESA_EASTING = 250668.73322714816
 ESA_NORTHING = 6259216.653115547
-META_DATA_COLUMNS = {
+_DATASET_COLUMNS_1 = {
+    # "Linear Power in the VV Polarization": "vv",
+    # "Linear Power in the VH Polarization": "vh",
+    "Thumbnail": "thumbnail",
+}
+_DATASET_COLUMNS_2 = {
     "Coastal aerosol": "B01",
     "Blue": "B02",
     "Green": "B03",
@@ -39,7 +49,11 @@ META_DATA_COLUMNS = {
     "Cloud Mask": "cloud_mask",
     "Thumbnail": "thumbnail",
 }
-
+DATASET_COLUMNS = {
+    "Core-S1RTC": _DATASET_COLUMNS_1,
+    "Core-S2L2A": _DATASET_COLUMNS_2,
+    "Core-S2L1C": _DATASET_COLUMNS_2,
+}
 DATA_PATH = Path(__file__).parent / "data"
 
 DESCRIPTION = f"""\
@@ -135,7 +149,7 @@ def _meta_data_path(dataset="Core-S2L2A", repository=REPOSITORY):
 
 
 def get_meta_data(dataset="Core-S2L2A", repository=REPOSITORY):
-    print(f"Loading {dataset}")
+    logging.info("Loading %s", dataset)
     path = _meta_data_path(dataset=dataset)
     if not path.exists():
         data = pd.read_parquet(_meta_data_url(dataset=dataset))
@@ -149,7 +163,7 @@ def get_meta_data(dataset="Core-S2L2A", repository=REPOSITORY):
     )
     # Optimize Performance
     data["timestamp"] = pd.to_datetime(data["timestamp"])
-    numeric_cols = ["cloud_cover", "nodata", "centre_lat", "centre_lon"]
+    numeric_cols = data.select_dtypes(include=["float64"]).columns
     data[numeric_cols] = data[numeric_cols].astype("float32")
 
     return data
@@ -158,7 +172,6 @@ def get_meta_data(dataset="Core-S2L2A", repository=REPOSITORY):
 def get_image(row, column="thumbnail"):
     parquet_url = row["parquet_url"]
     parquet_row = row["parquet_row"]
-    print(parquet_url, parquet_row, column)
     with open_parquet_file(parquet_url, columns=[column]) as f:
         with pq.ParquetFile(f) as pf:
             first_row_group = pf.read_row_group(parquet_row, columns=[column])
@@ -186,3 +199,12 @@ def get_closest_rows(data, target_easting, target_northing):
     )
     closest_rows = data[distance == distance.min()]
     return closest_rows
+
+
+def reconfig_basic_config(format_=FORMAT, level=logging.INFO):
+    """(Re-)configure logging"""
+    logging.basicConfig(format=format_, level=level, force=True)
+    logging.info("Logging.basicConfig completed successfully")
+
+
+reconfig_basic_config()