Filter out nonworking tasks from visualization

README.md

@@ -60,6 +60,7 @@ make help
 | **NOTION_OKR_LABELS**<br/>(optional) | *Requires: NOTION_KEY and NOTION_OKR_DATABASE_ID*<br/>The labels used in Notion. |
 | **NOTION_FINANCIAL_DATABASE_ID**<br/>(optional) | *Requires: NOTION_KEY*<br/>Like the example of *NOTION_OKR_DATABASE*, a database ID from notion is needed. The database should include the column names `Month`, `cost`, and `income`. |
 | **NOTION_WORKINGHOURS_DATABASE_ID**<br/>(optional) | *Requires: NOTION_KEY*<br/>Like the example of *NOTION_OKR_DATABASE*, a database ID from notion is needed. The database should include the column names `User`, `Daily`, `Delta`, `Start`, and `Stop`. |
+| **NOTION_TASKS_DATABASE_ID**<br/>(optional) | *Requires: NOTION_KEY*<br/>Like the example of *NOTION_OKR_DATABASE*, a database ID from notion is needed. The database should include the column names `TaskID`, `Tags`, `Desc`. |
 
 ## Configuration files
 

routes/index.py

@@ -10,7 +10,7 @@
 from dash import dcc, html
 
 from routes.date_utils import lookBack
-from routes.notion import OKR, Financials, WorkingHours
+from routes.notion import OKR, Financials, Tasks, WorkingHours
 from routes.tempo import SupplementaryData, TempoData
 
 # =========================================================
@@ -35,6 +35,7 @@
 NOTION_FINANCIAL_DATABASE_ID = os.environ.get("NOTION_FINANCIAL_DATABASE_ID", "")
 NOTION_WORKINGHOURS_DATABASE_ID = os.environ.get("NOTION_WORKINGHOURS_DATABASE_ID", "")
 NOTION_OKR_LABELS = [["2022", "Q4"], ["2022"]]
+NOTION_TASKS_DATABASE_ID = os.environ.get("NOTION_TASKS_DATABASE_ID", "")
 
 COLOR_HEAD = "#ad9ce3"
 COLOR_ONE = "#ccecef"
@@ -104,8 +105,18 @@ def tableHeight(table, base_height=208):
 # Fetch data
 # =========================================================
 
+
 # ---------------------------------------------------------
 # Data from NOTION
+
+if NOTION_KEY and NOTION_TASKS_DATABASE_ID:
+    tasks = Tasks(NOTION_KEY, NOTION_TASKS_DATABASE_ID)
+    tasks.get_tasks()
+    tasks_df = tasks.data
+    task_list = list(tasks_df["TaskID"])
+else:
+    task_list = []
+
 if NOTION_KEY and NOTION_FINANCIAL_DATABASE_ID:
     financials = Financials(NOTION_KEY, NOTION_FINANCIAL_DATABASE_ID)
     financials.get_financials()
@@ -120,6 +131,7 @@ def tableHeight(table, base_height=208):
 else:
     working_hours_df = pd.DataFrame()
 
+
 data = TempoData()
 data.load(from_date=START_DATE, to_date=YESTERDAY)
 
@@ -134,7 +146,8 @@ def tableHeight(table, base_height=208):
     table_missing_rates = data.missingRatesTable(tableHeight, COLOR_HEAD, COLOR_ONE)
 
 if not supplementary_data.working_hours.empty:
-    data.padTheData(supplementary_data.working_hours)
+    data.padTheData(supplementary_data.working_hours, task_list)
+    data.filter_data(task_list)
 
 
 # =========================================================

routes/notion.py

@@ -28,6 +28,27 @@ def fetch_data(self, database_id) -> requests.Response:
         return response
 
 
+class Tasks(Notion):
+    data: pd.DataFrame
+
+    def __init__(self, token: Optional[str] = None, database_id: str = "") -> None:
+        super().__init__(token, database_id)
+
+    def get_tasks(self) -> None:
+        result_dict = self.fetch_data(self.database_id).json()
+        data = pd.DataFrame(columns=["TaskID", "Tags", "Desc"])
+
+        for item in result_dict["results"]:
+            task_id = item["properties"]["TaskID"]["title"][0]["plain_text"]
+            tags = item["properties"]["Tags"]["multi_select"][0]["name"]
+            desc = item["properties"]["Desc"]["rich_text"][0]["plain_text"]
+
+            data.loc[-1] = [task_id, tags, desc]
+            data.index = data.index + 1
+
+        self.data = data.sort_values(by=["TaskID"])
+
+
 class WorkingHours(Notion):
     data: pd.DataFrame
 

routes/tempo.py

@@ -96,6 +96,7 @@ class TempoData:
     raw: pd.DataFrame
     data: pd.DataFrame
     padded_data: pd.DataFrame
+    filtered_data: pd.DataFrame
     this_year: int
     last_year: int
 
@@ -136,16 +137,18 @@ def getUsers(self) -> pd.Series:
 
     def byGroup(self) -> pd.DataFrame:
         """returns aggregated time and billable time grouped by date, user and group"""
-        return self.data.groupby(["Date", "User", "Group"], as_index=False)[["Time", "Billable"]].sum()
+        return self.padded_data.groupby(["Date", "User", "Group"], as_index=False)[["Time", "Billable"]].sum()
 
     def byTimeType(self) -> pd.DataFrame:
         """returns aggregated time and time type grouped by date, user and group"""
-        newdata = self.data.copy()
+        newdata = self.filtered_data.copy()
         newdata["Timetype"] = pd.isna(newdata["Rate"])
         newdata["Timetype"] = ["Billable" if not (x) else "Non-billable" for x in newdata["Timetype"]]
+
         newdata["Timetype"] = [
-            "VeriFriday" if x == "VF" else newdata["Timetype"][idx] for idx, x in enumerate(newdata["Group"])
+            "VeriFriday" if x == "VF" else newdata.iloc[idx]["Timetype"] for idx, x in enumerate(newdata["Group"])
         ]
+
         return newdata.groupby(["Date", "Timetype", "Group"], as_index=False)[["Time", "Timetype"]].sum()
 
     def byTotalGroup(self, days_back) -> pd.DataFrame:
@@ -340,15 +343,21 @@ def missingRatesTable(self, fnTableHeight=None, color_head="paleturquoise", colo
             fig.update_layout(height=fnTableHeight(rate_data))
         return fig
 
-    def padTheData(self, working_hours: pd.DataFrame) -> None:
+    def filter_data(self, task_list: list) -> None:
+        data = self.padded_data.copy()
+        data = data[~data["Key"].isin(task_list)]
+        self.filtered_data = data
+
+    def padTheData(self, working_hours: pd.DataFrame, task_list: list) -> None:
         """
         creates the self.padded_data padded with zero data
         for each User, an entry for the ZP group will be added for each date >= min(Date) && <= max(Date)
         Key: ZP-1, Time: 0, Billable: 0, Group: ZP, Internal: 0, Currency: EUR, Rate: 0, Income: 0
         """
-        self.padded_data = self.data
+        self.padded_data = self.data.copy()
+
         if not working_hours.empty:
-            for index, row in working_hours.iterrows():
+            for _, row in working_hours.iterrows():
                 df_user = pd.DataFrame()
                 user = row["User"]
                 if row["Start"] == "*":
@@ -371,6 +380,9 @@ def padTheData(self, working_hours: pd.DataFrame) -> None:
                 df_user["Rate"] = 0
                 df_user["Income"] = 0
                 self.padded_data = pd.concat([self.padded_data, df_user])
+            for idx, _ in self.padded_data.iterrows():
+                if self.padded_data.iloc[idx]["Key"] in task_list:
+                    self.padded_data.iloc[idx, self.padded_data.columns.get_loc("Group")] = "NW"
         else:
             for user in self.data["User"].unique():
                 df_user = pd.DataFrame()
@@ -388,10 +400,13 @@ def padTheData(self, working_hours: pd.DataFrame) -> None:
                 df_user["Rate"] = 0
                 df_user["Income"] = 0
                 self.padded_data = pd.concat([self.padded_data, df_user])
+            for idx, _ in self.padded_data.iterrows():
+                if self.padded_data.iloc[idx]["Key"] in task_list:
+                    self.padded_data.iloc[idx, self.padded_data.columns.get_loc("Group")] = "NW"
 
     def userRolling7(self, to_sum) -> pd.DataFrame:
         """returns rolling 7 day sums for Billable and non Billable time grouped by user"""
-        daily_sum = self.padded_data.groupby(["Date", "User"], as_index=False)[to_sum].sum()
+        daily_sum = self.filtered_data.groupby(["Date", "User"], as_index=False)[to_sum].sum()
         rolling_sum_7d = (
             daily_sum.set_index("Date").groupby(["User"], as_index=False).rolling("7d", min_periods=7)[to_sum].sum()
         )