add the algorithm to check changes

.gitlab-ci.yml

@@ -10,192 +10,95 @@ cache:
     - .pip-cache/
 
 stages:
-  - setup
+  - check_changes
   - process
-  - sync
   - deploy
 
 .install_dependencies:
   before_script:
-    - apt-get update && apt-get install -y curl
+    - apt-get update && apt-get install -y curl jq
     - pip install jupytext nbconvert
 
-setup:
-  stage: setup
-  extends: .install_dependencies
+check_changes:
+  stage: check_changes
   script:
-    - echo "Dependencies installed successfully"
+    - |
+      changes=$(git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA)
+      if echo "$changes" | grep -qvE "^src/"; then
+        echo "FULL_PROCESS=true" >> pipeline.env
+      else
+        echo "FULL_PROCESS=false" >> pipeline.env
+      fi
+      echo "$changes" > changed_files.txt
   artifacts:
+    reports:
+      dotenv: pipeline.env
     paths:
-      - .pip-cache/
+      - changed_files.txt
 
 process_notebooks:
   stage: process
   extends: .install_dependencies
   script:
     - |
-      for notebook in $(find ./src -name "*.ipynb"); do
-        clean_notebook=$(echo $notebook | sed 's/src/clean/')
-        mkdir -p $(dirname $clean_notebook)
-        cp $notebook $clean_notebook
-        jupyter nbconvert --ClearOutputPreprocessor.enabled=True --to notebook --inplace $clean_notebook
-        jupytext --to md $clean_notebook
-        jupytext --to py:percent $clean_notebook
-      done
-  artifacts:
-    paths:
-      - clean/
-
-create_sync_script:
-  stage: process
-  script:
-    - |
-      cat << EOF > sync_notebooks.py
-      import os
-      import sys
-      import jupytext
-      from nbformat import read, write, NO_CONVERT
-      import difflib
-
-      def sync_notebooks(source_dir, clean_dir):
-          for root, _, files in os.walk(source_dir):
-              for file in files:
-                  if file.endswith('.ipynb'):
-                      source_path = os.path.join(root, file)
-                      clean_path = source_path.replace(source_dir, clean_dir)
-                      md_path = clean_path.replace('.ipynb', '.md')
-                      py_path = clean_path.replace('.ipynb', '.py')
-                      
-                      # Regenerate parallel files if original nb changed
-                      if os.path.exists(source_path) and (not os.path.exists(clean_path) or 
-                                                          os.path.getmtime(source_path) > os.path.getmtime(clean_path)):
-                          print(f"Updating clean files from {source_path}")
-                          nb = read(source_path, as_version=NO_CONVERT)
-                          jupytext.write(nb, clean_path)
-                          jupytext.write(nb, md_path)
-                          jupytext.write(nb, py_path)
-                      
-                      # Check for edits in parallel files and update notebook
-                      elif os.path.exists(clean_path) and os.path.getmtime(clean_path) > os.path.getmtime(source_path):
-                          print(f"Checking for updates in clean files for {source_path}")
-                          source_nb = read(source_path, as_version=NO_CONVERT)
-                          clean_nb = jupytext.read(clean_path)
-                          
-                          if source_nb['cells'] != clean_nb['cells']:
-                              print(f"Conflicts detected in {file}")
-                              with open(f"{clean_path}.conflicts", 'w') as f:
-                                  f.write(f"Conflicts detected between {source_path} and {clean_path}\\n")
-                                  f.write("Diff:\\n")
-                                  for i, (s_cell, c_cell) in enumerate(zip(source_nb['cells'], clean_nb['cells'])):
-                                      if s_cell != c_cell:
-                                          f.write(f"Cell {i}:\\n")
-                                          diff = difflib.unified_diff(
-                                              s_cell['source'].splitlines(),
-                                              c_cell['source'].splitlines(),
-                                              fromfile='source',
-                                              tofile='clean',
-                                              lineterm=''
-                                          )
-                                          f.write('\\n'.join(diff))
-                                          f.write('\\n\\n')
-                          else:
-                              print(f"Updating source notebook from {clean_path}")
-                              write(clean_nb, source_path)
-
-      if __name__ == "__main__":
-          if len(sys.argv) != 3:
-              print("Usage: python sync_notebooks.py <source_dir> <clean_dir>")
-              sys.exit(1)
-          
-          source_dir = sys.argv[1]
-          clean_dir = sys.argv[2]
-          sync_notebooks(source_dir, clean_dir)
-      EOF
-  artifacts:
-    paths:
-      - sync_notebooks.py
-
-sync_notebooks:
-  stage: sync
-  extends: .install_dependencies
-  script:
-    - python sync_notebooks.py ./src ./clean
-    - |
-      if [ -n "$(find . -name '*.conflicts')" ]; then
-        echo "Conflicts detected. Please review the .conflicts files."
-        exit 1
+      if [ "$FULL_PROCESS" = "true" ]; then
+        for notebook in $(find ./src -name "*.ipynb"); do
+          clean_notebook=$(echo $notebook | sed 's/src/clean/')
+          mkdir -p $(dirname $clean_notebook)
+          cp $notebook $clean_notebook
+          jupyter nbconvert --ClearOutputPreprocessor.enabled=True --to notebook --inplace $clean_notebook
+          jupytext --to md $clean_notebook
+          jupytext --to py:percent $clean_notebook
+        done
+      else
+        echo "Skipping full processing"
       fi
   artifacts:
     paths:
       - src/
       - clean/
-      - "**/*.conflicts"
-    when: always
 
-.deploy_template:
+deploy:
   stage: deploy
+  extends: .install_dependencies
   script:
-    - echo "Deploying files from src directory"
-
-deploy-draft-students:
-  extends: .deploy_template
-  script:
-    - !reference [.deploy_template, script]
-    - |
-      curl -X POST "${WEBHOOK_URL}/files-sync-students-draft" \
-        -H "Content-Type: application/json" \
-        -H "X-Gitlab-Token: ${WEBHOOK_TOKEN}" \
-        -d '{
-          "object_kind": "pipeline",
-          "object_attributes": {
-            "status": "success",
-            "ref": "main"
-          }
-        }'
-  rules:
-    - if: $CI_COMMIT_BRANCH == "main"
-      changes:
-        - src/students/**/*
-      when: always
-
-deploy-teachers:
-  extends: .deploy_template
-  script:
-    - !reference [.deploy_template, script]
-    - |
-      curl -X POST "${WEBHOOK_URL}/files-sync-teachers" \
-        -H "Content-Type: application/json" \
-        -H "X-Gitlab-Token: ${WEBHOOK_TOKEN}" \
-        -d '{
-          "object_kind": "pipeline",
-          "object_attributes": {
-            "status": "success",
-            "ref": "main"
-          }
-        }'
-  rules:
-    - if: $CI_COMMIT_BRANCH == "main"
-      changes:
-        - src/teachers/**/*
-      when: always
-
-deploy-production-students:
-  extends: .deploy_template
-  script:
-    - !reference [.deploy_template, script]
     - |
-      curl -X POST "${WEBHOOK_URL}/files-sync-students" \
-        -H "Content-Type: application/json" \
-        -H "X-Gitlab-Token: ${WEBHOOK_TOKEN}" \
-        -d '{
-          "object_kind": "pipeline",
-          "object_attributes": {
-            "status": "success",
-            "ref": "release"
-          }
-        }'
-  rules:
-    - if: $CI_COMMIT_BRANCH == "release"
-      changes:
-        - src/students/**/*
-      when: always
\ No newline at end of file
+      changed_files=$(cat changed_files.txt | jq -R -s -c 'split("\n")')
+      full_process=$FULL_PROCESS
+      
+      deploy_payload=$(jq -n \
+        --arg kind "pipeline" \
+        --arg status "success" \
+        --arg ref "$CI_COMMIT_REF_NAME" \
+        --argjson changes "$changed_files" \
+        --argjson full_process $full_process \
+        '{
+          object_kind: $kind,
+          object_attributes: {
+            status: $status,
+            ref: $ref
+          },
+          changes: $changes,
+          full_process: $full_process
+        }')
+      
+      if [[ "$CI_COMMIT_BRANCH" == "main" && "$changed_files" == *"src/students/"* ]]; then
+        curl -X POST "${WEBHOOK_URL}/files-sync-students-draft" \
+          -H "Content-Type: application/json" \
+          -H "X-Gitlab-Token: ${WEBHOOK_TOKEN}" \
+          -d "$deploy_payload"
+      fi
+      
+      if [[ "$CI_COMMIT_BRANCH" == "main" && "$changed_files" == *"src/teachers/"* ]]; then
+        curl -X POST "${WEBHOOK_URL}/files-sync-teachers" \
+          -H "Content-Type: application/json" \
+          -H "X-Gitlab-Token: ${WEBHOOK_TOKEN}" \
+          -d "$deploy_payload"
+      fi
+      
+      if [[ "$CI_COMMIT_BRANCH" == "release" && "$changed_files" == *"src/students/"* ]]; then
+        curl -X POST "${WEBHOOK_URL}/files-sync-students" \
+          -H "Content-Type: application/json" \
+          -H "X-Gitlab-Token: ${WEBHOOK_TOKEN}" \
+          -d "$deploy_payload"
+      fi
\ No newline at end of file