Feat: Handle get_lot when RECAPITULATIF is nan

.drone.yml

@@ -12,7 +12,7 @@ steps:
     image: python:3.11
     commands:
       - echo ${DRONE_TAG}
-      - sed -i "s/VERSION_PLACEHOLDER/${DRONE_TAG}/g" pyproject.toml
+      - sed -i 's/version = "[^"]*"/version = "${DRONE_TAG}"/g' pyproject.toml
       - curl -sSL https://install.python-poetry.org | python3 -
       - export PATH="/root/.local/bin:$PATH"
       - poetry --version
@@ -22,6 +22,19 @@ steps:
       PYPI_TOKEN:
         from_secret: pypi_token
 
+  - name: Notify on matrix
+    image: plugins/matrix
+    environment:
+      MATRIX_ROOMID:
+        from_secret: MATRIX_ROOMID
+      MATRIX_ACCESSTOKEN: 
+        from_secret: MATRIX_ACCESSTOKEN
+      MATRIX_USERID:
+        from_secret: MATRIX_USERID
+    settings:
+      homeserver: https://matrix.poneyworld.net
+      template: "Une nouvelle version (${DRONE_TAG}) de pdf-oralia est publiée!"
+
     when:
       event:
         include:

pdf_oralia/extract.py

@@ -45,7 +45,7 @@ def from_pdf(pdf):
     charge_tables = []
     patrimoie_tables = []
 
-    for page in pdf.pages:
+    for page_number, page in enumerate(pdf.pages):
         page_text = page.extract_text()
         date = extract_date(page_text)
         additionnal_fields = {
@@ -76,7 +76,7 @@ def from_pdf(pdf):
             pass
 
         else:
-            raise ValueError("Page non reconnu")
+            logging.warning(f"Page {page_number+1} non reconnu. Page ignorée.")
 
     df_charge = charge.table2df(recapitulatif_tables + charge_tables)
     df_loc = locataire.table2df(loc_tables)

pdf_oralia/pages/charge.py

@@ -17,6 +17,7 @@ DF_TYPES = {
     "annee": str,
     "lot": str,
 }
+DEFAULT_FOURNISSEUR = "ROSIER MODICA MOTTEROZ SA"
 
 
 def is_it(page_text):
@@ -31,7 +32,10 @@ def is_it(page_text):
 def get_lot(txt):
     """Return lot number from "RECAPITULATIF DES OPERATIONS" """
     regex = r"[BSM](\d+)(?=\s*-)"
+    try:
         result = re.findall(regex, txt)
+    except TypeError:
+        return "*"
     if result:
         return "{:02d}".format(int(result[0]))
     return "*"
@@ -62,8 +66,8 @@ def extract(table, additionnal_fields: dict = {}):
             for k, v in additionnal_fields.items():
                 r[k] = v
 
-            if "honoraire" in row[RECAPITULATIF_DES_OPERATIONS]:
+            if "honoraire" in row[RECAPITULATIF_DES_OPERATIONS].lower():
-                r["Fournisseur"] = "IMI GERANCE"
+                r["Fournisseur"] = DEFAULT_FOURNISSEUR
 
             extracted.append(r)
 
@@ -83,6 +87,5 @@ def table2df(tables):
     df = pd.concat(dfs)
 
     df["immeuble"] = df["immeuble"].apply(lambda x: x[0].capitalize())
-    print(df.columns)
     df["lot"] = df["RECAPITULATIF DES OPERATIONS"].apply(get_lot)
-    return df.astype(DF_TYPES, errors="ignore")
+    return df.astype(DF_TYPES)

pdf_oralia/pages/locataire.py

@@ -1,3 +1,4 @@
+import numpy as np
 import pandas as pd
 
 DF_TYPES = {
@@ -33,7 +34,7 @@ def is_drop(row):
 
 
 def extract(table, additionnal_fields: dict = {}):
-    """Turn table to dictionary with additionnal fields"""
+    """Turn table to dictionary with additional fields"""
     extracted = []
     header = table[0]
     for row in table[1:]:
@@ -138,8 +139,6 @@ def join_row(table):
                     }
                 )
                 joined.append(row)
-            else:
-                pass
 
     return joined
 
@@ -159,4 +158,9 @@ def table2df(tables):
     df["immeuble"] = df["immeuble"].apply(lambda x: x[0].capitalize())
     df["Type"] = df["Type"].apply(clean_type)
 
-    return df.astype(DF_TYPES, errors="ignore")
+    numeric_cols = [k for k, v in DF_TYPES.items() if v == float]
+    df[numeric_cols] = df[numeric_cols].replace("", np.nan)
+
+    df = df.drop(df[(df["Locataires"] == "") & (df["Période"] == "")].index)
+
+    return df.astype(DF_TYPES)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pdf-oralia"
-version = "1.dev"
+version = "dev"
 description = ""
 authors = ["Bertrand Benjamin <benjamin.bertrand@opytex.org>"]
 readme = "README.md"