Update notebooks: new ADML sensitivity-analysis notebooks, PM-4 simulation code, notebook fixes

.github/workflows/check-R-notebooks.yml

@@ -15,8 +15,9 @@ jobs:
   test-R-notebooks:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        directory: ['PM1', 'PM2', 'PM3', 'PM4', 'PM5', 'CM1', 'CM2', 'CM3', 'AC1', 'AC2', 'T']
+        directory: ['PM1', 'PM2', 'PM3', 'PM4', 'PM5', 'CM1', 'CM2', 'CM3', 'AC1', 'AC2', 'T', 'T-ADML']
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
@@ -37,21 +38,21 @@ jobs:
             grep -E '^${{ matrix.directory }}/[^/]+\.Rmd$' changed_files.txt
             exit 1
           fi
-          grep -E '^${{ matrix.directory }}/[^/]+\.irnb$|^${{ matrix.directory }}/.*\.R|\.github/workflows/check-and-transform-R-notebooks.yml$' changed_files.txt > changed_notebooks.txt || echo "No notebooks changed" > changed_notebooks.txt
+          grep -E '^${{ matrix.directory }}/[^/]+\.irnb$|^${{ matrix.directory }}/.*\.R|\.github/workflows/check-R-notebooks.yml$' changed_files.txt > changed_notebooks.txt || echo "No notebooks changed" > changed_notebooks.txt
 
       - name: Find changed notebooks in Push
         if: github.event_name == 'push'
         id: find_notebooks_push
         run: |
           git diff --name-only ${{ github.event.before }} ${{ github.event.after }} > changed_files.txt
-          grep  -E '^${{ matrix.directory }}/[^/]+\.irnb$|^${{ matrix.directory }}/.*\.R|\.github/workflows/check-and-transform-R-notebooks.yml$' changed_files.txt > changed_notebooks.txt || echo "No notebooks changed" > changed_notebooks.txt
+          grep  -E '^${{ matrix.directory }}/[^/]+\.irnb$|^${{ matrix.directory }}/.*\.R|\.github/workflows/check-R-notebooks.yml$' changed_files.txt > changed_notebooks.txt || echo "No notebooks changed" > changed_notebooks.txt
 
       - name: Check if any notebooks changed in PR or Push
         if: (github.event_name == 'push') || (github.event_name == 'pull_request')
         id: check_notebooks
         run: |
           cat changed_notebooks.txt
-          if grep -q -E '^${{ matrix.directory }}/[^/]+\.irnb$|^${{ matrix.directory }}/.*\.R|\.github/workflows/check-and-transform-R-notebooks.yml$' changed_notebooks.txt; then
+          if grep -q -E '^${{ matrix.directory }}/[^/]+\.irnb$|^${{ matrix.directory }}/.*\.R|\.github/workflows/check-R-notebooks.yml$' changed_notebooks.txt; then
             echo "notebooks_changed=true" >> $GITHUB_ENV
           else
             echo "notebooks_changed=false" >> $GITHUB_ENV
@@ -70,7 +71,7 @@ jobs:
         if: env.notebooks_changed == 'true'
         run: |
           sudo apt-get update
-          sudo apt-get install -y libcurl4-openssl-dev
+          sudo apt-get install -y libcurl4-openssl-dev libuv1-dev libglpk-dev
 
       - name: Set up Python
         if: env.notebooks_changed == 'true'
@@ -125,7 +126,9 @@ jobs:
             library(lintr)
             linters <- linters_with_defaults(line_length_linter = line_length_linter(120),
                                              object_name_linter = object_name_linter(styles = c("snake_case", "CamelCase", "camelCase")),
-                                             object_usage_linter = NULL)
+                                             object_usage_linter = NULL,
+                                             pipe_consistency_linter = NULL,
+                                             return_linter = NULL)
             rmd_files <- list.files(path = "${{ matrix.directory }}", pattern = "\\.Rmd$", full.names = TRUE)
             linting_error <- FALSE
             for (rfile in rmd_files) {
@@ -195,6 +198,8 @@ jobs:
           ' 2>/dev/null
         env:
           GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+          # let R (reticulate/keras) use the Python with TensorFlow installed above
+          RETICULATE_PYTHON: ${{ env.pythonLocation }}/bin/python
 
       - name: Upload execution log
         if: "((env.notebooks_changed == 'true') && (env.linting_errors_found == 'false'))"

.github/workflows/check-python-notebooks.yml

@@ -15,10 +15,11 @@ jobs:
   test-python-notebooks:
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
         # Removed: {windows-latest, macos-latest} as they were taking too long. Consider adding back in the future
         os: [ubuntu-latest]
-        directory: [PM1, PM2, PM3, PM4, PM5, CM1, CM2, CM3, AC1, AC2, T]
+        directory: [PM1, PM2, PM3, PM4, PM5, CM1, CM2, CM3, AC1, AC2, T, T-ADML]
 
     steps:
     - name: Checkout repository
@@ -34,21 +35,21 @@ jobs:
         # git fetch origin ${{ github.event.pull_request.base.ref }} ${{ github.event.pull_request.head.ref }}
         # git diff --name-only origin/${{ github.event.pull_request.base.ref }}...origin/${{ github.event.pull_request.head.ref }} > changed_files.txt
         git diff --name-only -r HEAD^1 HEAD > changed_files.txt
-        grep -E '\.ipynb$|\.py$|\.github/workflows/python-notebooks.yml$' changed_files.txt > changed_notebooks.txt || echo "No notebooks changed" > changed_notebooks.txt
+        grep -E '\.ipynb$|\.py$|\.github/workflows/check-python-notebooks.yml$' changed_files.txt > changed_notebooks.txt || echo "No notebooks changed" > changed_notebooks.txt
 
     - name: Find changed notebooks in Push
       if: github.event_name == 'push'
       id: find_notebooks_push
       run: |
         git diff --name-only ${{ github.event.before }} ${{ github.event.after }} > changed_files.txt
-        grep -E '\.ipynb$|\.py$|\.github/workflows/python-notebooks.yml$' changed_files.txt > changed_notebooks.txt || echo "No notebooks changed" > changed_notebooks.txt
+        grep -E '\.ipynb$|\.py$|\.github/workflows/check-python-notebooks.yml$' changed_files.txt > changed_notebooks.txt || echo "No notebooks changed" > changed_notebooks.txt
 
     - name: Check if any notebooks changed in PR or Push
       if: (github.event_name == 'push') || (github.event_name == 'pull_request')
       id: check_notebooks
       run: |
         cat changed_notebooks.txt
-        if grep -q -E '^${{ matrix.directory }}/[^/]+\.ipynb$|^${{ matrix.directory }}/.*\.py$|\.github/workflows/python-notebooks.yml$' changed_notebooks.txt; then
+        if grep -q -E '^${{ matrix.directory }}/[^/]+\.ipynb$|^${{ matrix.directory }}/.*\.py$|\.github/workflows/check-python-notebooks.yml$' changed_notebooks.txt; then
           echo "notebooks_changed=true" >> $GITHUB_ENV
         else
           echo "notebooks_changed=false" >> $GITHUB_ENV
@@ -81,12 +82,9 @@ jobs:
       run: sudo apt-get install -y libgraphviz-dev
       shell: bash
 
-    - name: Install R if folder is CM3
-      if: "(env.notebooks_changed == 'true') && (matrix.directory == 'CM3' && matrix.os == 'ubuntu-latest')"
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y r-base
-      shell: bash
+    - name: Install R if folder is CM3 or T-ADML
+      if: "(env.notebooks_changed == 'true') && ((matrix.directory == 'CM3' || matrix.directory == 'T-ADML') && matrix.os == 'ubuntu-latest')"
+      uses: r-lib/actions/setup-r@v2
 
     - name: Set up Python
       if: "(env.notebooks_changed == 'true') && (! (matrix.directory == 'CM3' && (matrix.os == 'windows-latest' || matrix.os == 'macos-latest')))"
@@ -117,6 +115,8 @@ jobs:
         flake8_errors=0
         for notebook in ${{ matrix.directory }}/*.ipynb; do
           [ "$(basename "$notebook")" = "python-dosearch.ipynb" ] && continue
+          # uses %%R (rpy2) cells, which flake8 cannot parse
+          [ "$(basename "$notebook")" = "python-ADML-401k.ipynb" ] && continue
           flake8-nb --config=.flake8 "$notebook" > "linting_logs/$(basename "$notebook" .ipynb)_linting.txt" 2>&1 || flake8_errors=$((flake8_errors+1))
         done
         echo "flake8_errors=$flake8_errors" >> $GITHUB_ENV

.gitignore

@@ -132,3 +132,8 @@ dmypy.json
 /CM1/hdmpy
 /AC1/hdmpy
 /PM2/hdmpy
+
+# Notebook updates
+/TEMP_updated_notebooks
+/tools
+.vscode/settings.json

AC1/python-proxy-controls.ipynb

@@ -611,8 +611,10 @@
     "data = data[data.wt != 999]\n",
     "data = data[data.parity != 99]\n",
     "data = data[data.parity != 9]\n",
+    "data = data[data.race != 99]\n",
     "data = data[np.logical_and(data.number != 98, data.number != 99)]\n",
     "data = data[np.logical_and(data.inc != 98, data.inc != 99)]\n",
+    "data = data[data.age != 99]\n",
     "data.shape"
    ]
   },
@@ -624,7 +626,7 @@
    },
    "outputs": [],
    "source": [
-    "X = np.array(patsy.dmatrix('0 + C(race) + age + C(sex)', data))\n",
+    "X = np.array(patsy.dmatrix('0 + C(race) + age', data))\n",
     "D = np.array(patsy.dmatrix('0 + number', data))\n",
     "Q = np.array(patsy.dmatrix('0 + C(ed)', data))\n",
     "S = np.array(patsy.dmatrix('0 + parity', data))\n",

AC1/python-sensitivity-analysis-with-sensmakr-and-debiased-ml.ipynb

@@ -318,6 +318,8 @@
    },
    "outputs": [],
    "source": [
+    "import sys\n",
+    "sys.path.insert(1, \"./hdmpy\")\n",
     "import hdmpy\n",
     "\n",
     "\n",

AC1/r-proxy-controls.irnb

@@ -328,6 +328,9 @@
    "source": [
     "# Filter data to exclude entries where income, number of cigarettes smoked,\n",
     "# race, age are not asked or not known\n",
+    "data <- data[data$wt != 999, ]\n",
+    "data <- data[data$parity != 99, ]\n",
+    "data <- data[data$parity != 9, ]\n",
     "data <- data[data$race != 99, ]\n",
     "data <- data[!(data$number %in% c(98, 99)), ]\n",
     "data <- data[!(data$inc %in% c(98, 99)), ]\n",