Merge pull request #4650 from mathesar-foundation/release-0.4.0

Release 0.4.0

Author: zackkrida

.env.example

@@ -1,2 +1,3 @@
 ALLOWED_HOSTS=".localhost, 127.0.0.1, [::1]"
 SECRET_KEY="2gr6ud88x=(p855_5nbj_+7^bw-iz&n7ldqv%94mjaecl+b9=4"
+OIDC_CONFIG_DICT="{\"version\": 1,\"oidc_providers\": {  \"provider1\": {    \"provider_name\": \"okta\",    \"client_id\": \"client-id\",    \"secret\": null,    \"server_url\": \"https://trial-2872264-admin.okta.com\"  }}}"

.github/ISSUE_TEMPLATE/bug_report.md

@@ -14,6 +14,7 @@ labels: "type: bug, needs: triage"
 <!-- How can we recreate this bug? Please try to provide a Minimal, Complete, and Verifiable (http://stackoverflow.com/help/mcve) example if code-related. -->
 
 ## Environment
+ - Mathesar version: (Visit https://{your-mathesar-install.com}/administration/update/ to find the current version)
  - OS: (_eg._ macOS 10.14.6; Fedora 32)
  - Browser: (_eg._ Safari; Firefox)
  - Browser Version: (_eg._ 13; 73)

.github/ISSUE_TEMPLATE/roadmap-item.md

@@ -0,0 +1,20 @@
+---
+name: "[MAINTAINERS ONLY] Roadmap item"
+about: "This issue template is for the convenience of Mathesar maintainers. Please do not use it if you are not a maintainer."
+labels: "type: project, needs: requirements"
+---
+
+>  _This is a meta issue tracking the implementation of this project._
+
+## Outcome
+<!-- A clear and concise description of the outcome -->
+
+More detail:
+- Requirements TBD
+- Project proposal TBD
+
+## Tasks
+*Issues to be created soon.*
+
+## Related
+<!-- Any related resources, discussions, issues, etc. -->

.gitignore

@@ -181,6 +181,9 @@ Temporary Items
 .media/
 docs/_build/
 
+## SSO-SPECIFIC ##
+sso.yml
+
 # UI
 node_modules/
 

DEVELOPER_GUIDE.md

@@ -49,28 +49,9 @@ Before getting started with your code changes, read our [Contributor guide](./CO
 
 ## Loading sample data
 
-- Using a CSV File (limited visibility of features):
+Mathesar comes with several built-in sample datasets to help you get started quickly with local development or feature evaluation. You can install these directly from the "Create" and "Connect Database" dialogs in the UI. Each dataset is designed to showcase different relationships (many-to-many, many-to-one) in commonplace example scenarios. The **Library Management** dataset is the largest, and is especially useful for performance-related testing.
 
-    For sample table data, you can create a new table in the UI using the `patents.csv` file found in `/mathesar/tests/data`.
-
-- Using Mathesar Data Playground (recommended):
-
-    1. Clone the `mathesar-data-playground` repo:
-        ```
-        git clone https://github.com/mathesar-foundation/mathesar-data-playground.git
-        ```
-
-    2. Load the data from sql by running:
-        ```
-        sudo docker exec -i mathesar_dev_db bash -c 'psql -U mathesar' < /path/to/your/cloned/repo/mathesar-data-playground/realistic_library_simulation/simulation_runs/simulation_run_20230106_00.sql
-        ```
-        ```
-        sudo docker exec -i mathesar_dev_db bash -c 'psql -U mathesar' < /path/to/your/cloned/repo/mathesar-data-playground/realistic_library_simulation/simulation_runs/simulation_run_20230106_00_checkouts.sql
-        ```
-    3. [Sync]( https://docs.mathesar.org/user-guide/syncing-db/) these changes from the UI.
-
-
-<!-- TODO add more content about sample data -->
+For more example datasets, see the [mathesar-data-playground](https://github.com/mathesar-foundation/mathesar-data-playground) repository. It includes both the preinstalled datasets and additional ones, like a large, high-quality **Movies** dataset. [Installation instructions for the Movies dataset are available here](https://github.com/mathesar-foundation/mathesar-data-playground/tree/master/movie_rentals#how-to-load-this-data-into-mathesar).
 
 ## API
 

README.md

@@ -9,7 +9,7 @@
 </p>
 
 <p align="center">
-  <a href="https://mathesar.org?ref=github-readme" target="_blank">Website</a> • <a href="https://docs.mathesar.org?ref=github-readme-top" target="_blank">Docs</a> • <a href="https://wiki.mathesar.org/en/community/matrix" target="_blank">Matrix (chat)</a> • <a href="https://discord.gg/enaKqGn5xx" target="_blank">Discord</a> • <a href="https://wiki.mathesar.org/" target="_blank">Contributor Wiki</a>
+  <a href="https://mathesar.org?ref=github-readme" target="_blank">Website</a> • <a href="https://docs.mathesar.org?ref=github-readme-top" target="_blank">Docs</a> • <a href="https://wiki.mathesar.org/en/community/matrix" target="_blank">Matrix (chat)</a> • <a href="https://discord.gg/enaKqGn5xx" target="_blank">Discord</a> • <a href="https://wiki.mathesar.org/" target="_blank">Contributor Wiki</a> • <a href="https://github.com/orgs/mathesar-foundation/projects/2" target="_blank">Roadmap</a>
 </p>
 
 

config/settings/common_settings.py

@@ -11,8 +11,12 @@
 """
 
 import os
+import traceback
 from pathlib import Path
 
+import json
+import yaml
+from collections import defaultdict
 from config.database_config import PostgresConfig, parse_port
 
 
@@ -34,6 +38,10 @@
     "django_property_filter",
     "modernrpc",
     "mathesar",
+    "allauth",
+    "allauth.account",
+    "allauth.socialaccount",
+    "allauth.socialaccount.providers.openid_connect",
 ]
 
 MIDDLEWARE = [
@@ -48,8 +56,98 @@
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
     "mathesar.middleware.CursorClosedHandlerMiddleware",
     "mathesar.middleware.PasswordChangeNeededMiddleware",
+    "allauth.account.middleware.AccountMiddleware",
 ]
 
+OIDC_CONFIG_FILE = BASE_DIR.joinpath('sso.yml')
+OIDC_CONFIG_DICT = {}
+# Try loading OIDC_CONFIG_DICT from env, iff it doesn't exist, try loading from sso.yml
+try:
+    OIDC_CONFIG_DICT = json.loads(os.getenv('OIDC_CONFIG_DICT', "{}"))
+except Exception as e:
+    traceback.print_exception(type(e), e, e.__traceback__)
+try:
+    if OIDC_CONFIG_DICT in [None, {}] and OIDC_CONFIG_FILE.exists():
+        with open(OIDC_CONFIG_FILE, "rb") as f:
+            OIDC_CONFIG_DICT = yaml.full_load(f)
+except Exception as e:
+    traceback.print_exception(type(e), e, e.__traceback__)
+OIDC_CONFIG = []
+OIDC_ALLOWED_EMAIL_DOMAINS = {}
+OIDC_DEFAULT_PG_ROLE_MAP = defaultdict(list)
+
+try:
+    for providers, config in OIDC_CONFIG_DICT['oidc_providers'].items():
+        if not config:
+            continue
+        provider_name = config.get("provider_name")
+        client_id = config.get("client_id")
+        secret = config.get("secret")
+        server_url = config.get("server_url")
+        allowed_email_domains = config.get("allowed_email_domains", [])  # Here [] means allow all domains.
+        default_pg_role = config.get("default_pg_role", {})
+        if all([provider_name, client_id, secret, server_url]):
+            OIDC_CONFIG.append(
+                {
+                    "provider_id": provider_name.lower(),
+                    "name": provider_name.lower(),
+                    "client_id": client_id,
+                    "secret": secret,
+                    "settings": {
+                        "server_url": server_url
+                    }
+                }
+            )
+
+        if isinstance(allowed_email_domains, list):
+            OIDC_ALLOWED_EMAIL_DOMAINS[provider_name] = allowed_email_domains
+        elif isinstance(allowed_email_domains, str):
+            OIDC_ALLOWED_EMAIL_DOMAINS[provider_name] = [allowed_email_domains]
+
+        for role_info in default_pg_role.values():
+            if not role_info:
+                continue
+            db_name = role_info.get("name")
+            host = role_info.get("host")
+            port = role_info.get("port")
+            role_name = role_info.get("role")
+            if all([db_name, host, port, role_name]):
+                OIDC_DEFAULT_PG_ROLE_MAP[provider_name].append(
+                    {
+                        "db_name": db_name,
+                        "host": host,
+                        "port": port,
+                        "role_name": role_name
+                    }
+                )
+except Exception as e:
+    # We swallow any exceptions when SSO is misconfigured in sso.yml so that the django server doesn't fail to start.
+    if OIDC_CONFIG_DICT not in [None, {}]:
+        traceback.print_exception(type(e), e, e.__traceback__)  # Print the traceback in case of an exception.
+
+
+SOCIALACCOUNT_PROVIDERS = {
+    "openid_connect": {
+        "APPS": OIDC_CONFIG
+    }
+}
+
+SOCIALACCOUNT_ADAPTER = "mathesar.oidc.SocialAccountAdapter"
+
+AUTHENTICATION_BACKENDS = [
+    # Needed to login by username in Django admin, regardless of `allauth`
+    'django.contrib.auth.backends.ModelBackend',
+
+    # `allauth` specific authentication methods, such as login by email
+    'allauth.account.auth_backends.AuthenticationBackend',
+]
+
+# Allows us to merge existing users with OIDC logins.
+# More context: https://docs.allauth.org/en/dev/socialaccount/configuration.html
+SOCIALACCOUNT_EMAIL_AUTHENTICATION = True
+SOCIALACCOUNT_EMAIL_AUTHENTICATION_AUTO_CONNECT = True
+
+
 ROOT_URLCONF = "config.urls"
 
 MODERNRPC_METHODS_MODULES = [
@@ -117,6 +215,9 @@
         role=POSTGRES_USER,
         password=POSTGRES_PASSWORD,
     ).to_django_dict()
+    DATABASES['default']['OPTIONS'] = {
+        "application_name": "Mathesar Django"
+    }
 
 for db_key, db_dict in DATABASES.items():
     # Engine should be '.postgresql' or '.postgresql_psycopg2' for all db(s)

db/connection.py

@@ -1,3 +1,4 @@
+import psycopg
 from psycopg.rows import dict_row
 from uuid import uuid4
 
@@ -58,3 +59,14 @@ def select_from_msar_func(conn, func_name, *args):
 def load_file_with_conn(conn, file_handle):
     """Run an SQL script from a file, using psycopg."""
     conn.execute(file_handle.read())
+
+
+def mathesar_connection(*args, **kwargs):
+    """
+    All Mathesar psycopg connections should use this function. It adds an
+    application_name to each connection (prepending any previously-set
+    application name). This is visible in the `pg_stat_activity` table,
+    useful for debugging (and perhaps required by some DBAs).
+    """
+    kwargs.update(application_name="Mathesar " + kwargs.get("application_name", ""))
+    return psycopg.connect(*args, **kwargs)

db/deprecated/engine.py

@@ -45,6 +45,10 @@ def create_engine(conn_url, *args, **kwargs):
     across all engines. This is important for testing: without this intervention, fixtures become
     randomly corrupted.
     """
+    kwargs.update(
+        connect_args={"application_name": "Mathesar db.deprecated.engine.create_future_engine"},
+        pool_size=2,
+    )
     engine = sa_create_engine(conn_url, *args, **kwargs)
     _make_ischema_names_unique(engine)
     return engine

db/deprecated/utils.py

@@ -1,11 +1,12 @@
 import inspect
 import warnings
 
-import psycopg
 from psycopg2 import errors as p_errors
 import sqlalchemy
 from sqlalchemy.exc import ProgrammingError
 
+from db.connection import mathesar_connection
+
 
 class UndefinedFunction(Exception):
     pass
@@ -74,10 +75,11 @@ def get_pg_catalog_table(table_name, engine, metadata):
 
 
 def engine_to_psycopg_conn(engine):
-    return psycopg.connect(
+    return mathesar_connection(
         host=engine.url.host or engine.url.query["host"],
         port=engine.url.port,
         dbname=engine.url.database,
         user=engine.url.username,
-        password=engine.url.password
+        password=engine.url.password,
+        application_name='mathesar.db.deprecated.utils.engine_to_psycopg_conn',
     )