v5 support

Dockerfile.jinja

@@ -1,5 +1,5 @@
 # basic python3 image as base
-FROM harbor2.vantage6.ai/infrastructure/algorithm-base
+FROM harbor2.vantage6.ai/infrastructure/algorithm-base:5.0
 
 # This is a placeholder that should be overloaded by invoking
 # docker build with '--build-arg PKG_NAME=...'
@@ -9,23 +9,6 @@ ARG PKG_NAME="{{algorithm_name}}"
 COPY . /app
 RUN pip install /app
 
-{% if use_vpn %}
-# Specify the ports that are used for VPN communication, along with a label
-# that helps you identify them. As an example, port 8888 is used here. The label
-# must be specified as the port number with a 'p' prefix, e.g. 'p8888'.
-    {% if vpn_expose %}
-        {%- for port_dict in vpn_expose %}
-EXPOSE {{port_dict.port}}
-LABEL p{{port_dict.port}} = '{{port_dict.label}}'
-        {% endfor %}
-    {% else %}
-# TODO provide a sensible label below. Feel free to add more ports if needed by
-# adding additional EXPOSE and LABEL commands.
-EXPOSE 8888
-LABEL p8888='some_label'
-    {% endif %}
-{% endif %}
-
 # Set environment variable to make name of the package available within the
 # docker image.
 ENV PKG_NAME=${PKG_NAME}

README.md.jinja

@@ -18,15 +18,16 @@ Please ensure to execute the following steps. The steps are also indicated with
 TODO statements in the generated code - so you can also simply search the
 code for TODO instead of following the checklist below.
 
-- [ ] Include a URL to your code repository in setup.py.
+- [ ] Fill out the fields in the `pyproject.toml` file, such as a URL to your code
+      repository. Alternatively, remove these fields.
 - [ ] Implement your algorithm functions.
   - [ ] You are free to add more arguments to the functions. Be sure to add them
     *after* the `client` and dataframe arguments.
   - [ ] When adding new arguments, if you run the `test/test.py` script, be sure
     to include values for these arguments in the `client.task.create()` calls
     that are available there.
 - [ ] If you are using Python packages that are not in the standard library, add
-  them to the `requirements.txt` and `setup.py` file.
+  them to the `pyproject.toml` file. Note that `pandas` is already included by default.
 {% if has_docs %}
 - [ ] Fill in the documentation template. This will help others to understand your
   algorithm, be able to use it safely, and to contribute to it.
@@ -37,15 +38,16 @@ code for TODO instead of following the checklist below.
 {% endif %}
 - [ ] If you want to submit your algorithm to a vantage6 algorithm store, be sure
   to fill in everything in ``algorithm_store.json`` (and be sure to update
-  it if you change function names, arguments, etc.).
+  it if you change function names, arguments, etc.). It is recommended to run
+  ``v6 algorithm generate-store-json`` to automatically generate the file - this
+  should work especially well if you have added proper docstrings to your functions.
+  Note that you do need the `vantage6` CLI to be able to use this command, which can be
+  installed by e.g. running `pip install vantage6` (or `uv pip install vantage6`).
 {% if has_gh_pipeline %}
 - [ ] Create a ``DOCKER_USER`` and ``DOCKER_PASSWORD`` secret in the GitHub repository
   settings. This will be used to push the Docker image to the registry in the github
   pipeline.
 {% endif %}
-{% if use_vpn %}
-- [ ] Review the EXPOSE and LABEL commands in the Dockerfile for VPN
-{% endif %}
 - [ ] Finally, remove this checklist section to keep the README clean.
 
 ### Dockerizing your algorithm

algorithm_store.json.jinja

@@ -1,7 +1,7 @@
 {
   "name": "{{algorithm_name}}",
   "image": "{{docker_image}}",
-  "vantage6_version": "4.6",
+  "vantage6_version": "5.0",
   "code_url": "https://mygitrepo.org",
   "documentation_url": "",
   "partitioning": "horizontal",
@@ -10,17 +10,12 @@
     {
       "name": "{{central_function_name}}",
       "description": "{{algorithm_description}}",
-      "type": "central",
+      "step_type": "central_compute",
       "databases": [
-        {%- for idx in range(partial_function_number_databases) -%}
+        {%- for idx in range(federated_function_number_databases) -%}
           {
-          "name": "Partial database {{idx + 1}}"
-          }{%- if not loop.last or central_function_number_databases > 0 -%},{%- endif -%}
-        {% endfor %}
-        {% for idx in range(central_function_number_databases) %}
-        {
-        "name": "Central database {{idx + 1}}"
-        }{%- if not loop.last -%},{%- endif -%}
+          "name": "Central database {{idx + 1}}"
+          }{%- if not loop.last or federated_function_number_databases > 0 -%},{%- endif -%}
         {% endfor %}
       ],
       "arguments": [
@@ -33,21 +28,21 @@
         {% endfor %}
       ]
     }{%- endif -%}
-    {%- if has_central_function and has_partial_function -%},{%- endif -%}
-    {% if has_partial_function -%}
+    {%- if has_central_function and has_federated_function -%},{%- endif -%}
+    {% if has_federated_function -%}
     {
-      "name": "{{partial_function_name}}",
+      "name": "{{federated_function_name}}",
       "description": "",
-      "type": "federated",
+      "step_type": "federated_compute",
       "databases": [
-        {%- for idx in range(partial_function_number_databases) -%}
+        {%- for idx in range(federated_function_number_databases) -%}
           {
           "name": "Database {{idx + 1}}"
           }{%- if not loop.last -%},{%- endif -%}
         {% endfor %}
       ],
       "arguments": [
-        {%- for arg in partial_args -%}
+        {%- for arg in federated_args -%}
         {
           "name": "{{arg}}",
           "type": "",

cleanup.py

@@ -24,10 +24,12 @@ def cleanup() -> None:
         print("Removing LICENSE file as no license was chosen...")
         Path("LICENSE").unlink()
 
-    # Remove partial function files if partial function is not defined
-    if not copier_config.get("has_partial_function"):
-        print("Removing partial function file as partial function is not defined...")
-        Path(algorithm_name, "partial.py").unlink()
+    # Remove federated function files if federated function is not defined
+    if not copier_config.get("has_federated_function"):
+        print(
+            "Removing federated function file as federated function is not defined..."
+        )
+        Path(algorithm_name, "federated.py").unlink()
 
     # Remove central function files if central function is not defined
     if not copier_config.get("has_central_function"):

copier.yml

@@ -30,86 +30,103 @@ central_function_name:
   type: str
   when: "{{ has_central_function }}"
   help: "What is the name of your central function?"
-  default: central
+  default: central_function
 
-central_function_client:
-  type: bool
+central_args:
+  type: json
   when: "{{ has_central_function }}"
-  help: "Do you want to use a client in your central function?"
+  help: "Add a list of arguments to the central function '{{ central_function_name }}'"
+  default: ['arg1', ]
+  multiline: true
+
+# ------------- Define federated algorithm function ---------- #
+
+has_federated_function:
+  type: bool
+  help: "Do you want to use a federated function in your algorithm?"
   default: true
 
-central_function_data:
+federated_function_name:
+  type: str
+  when: "{{ has_federated_function }}"
+  help: "What is the name of your federated function?"
+  default: federated_function
+
+federated_function_data:
   type: bool
-  when: "{{ has_central_function }}"
-  help: "Do you want to use data in your central function?"
-  default: false
+  when: "{{ has_federated_function }}"
+  help: "Do you want to use data in your federated function?"
+  default: true
 
-central_function_number_databases:
+federated_function_number_databases:
   type: int
-  when: "{{ has_central_function and central_function_data }}"
-  help: "How many databases do you want to use in your central function?"
+  when: "{{ has_federated_function and federated_function_data }}"
+  help: "How many databases do you want to use in your federated function?"
   default: |-
-    {%- if has_central_function and central_function_data -%}
+    {%- if has_federated_function and federated_function_data -%}
       1
     {%- else -%}
       0
     {%- endif -%}
   validator: |-
-    {% if central_function_data and central_function_number_databases < 1 %}
+    {% if has_federated_function and federated_function_number_databases < 1 %}
       Must be at least 1
     {% endif %}
 
-central_args:
+federated_args:
   type: json
-  when: "{{ has_central_function }}"
-  help: "Add a list of arguments to the central function '{{ central_function_name }}'"
+  when: "{{ has_federated_function }}"
+  help: "Add a list of arguments to the federated function '{{ federated_function_name }}'"
   default: ['arg1', ]
   multiline: true
 
-# ------------- Define partial algorithm function ---------- #
+# ----------------------- Data extraction ------------------- #
 
-has_partial_function:
+import_infra_extraction:
   type: bool
-  help: "Do you want to use a partial function in your algorithm?"
+  help: "Do you want to make the default vantage6 data extraction functions available in your algorithm?"
   default: true
 
-partial_function_name:
+has_data_extraction:
+  type: bool
+  help: "Do you want to define your own data extraction function in your algorithm?"
+  default: true
+
+data_extraction_function_name:
   type: str
-  when: "{{ has_partial_function }}"
-  help: "What is the name of your partial function?"
-  default: partial
+  when: "{{ has_data_extraction }}"
+  help: "What is the name of your data extraction function?"
+  default: data_extraction_function
 
-partial_function_client:
+data_extraction_args:
+  type: json
+  when: "{{ has_data_extraction }}"
+  help: "Add a list of arguments to the data extraction function '{{ data_extraction_function_name }}'"
+  default: ['arg1', ]
+  multiline: true
+
+# ----------------------- Data preprocessing ------------------- #
+
+import_infra_preprocessing:
   type: bool
-  when: "{{ has_partial_function }}"
-  help: "Do you want to use a client in your partial function?"
-  default: false
+  help: "Do you want to make the default vantage6 data preprocessing functions available in your algorithm?"
+  default: true
 
-partial_function_data:
+has_data_preprocessing:
   type: bool
-  when: "{{ has_partial_function }}"
-  help: "Do you want to use data in your partial function?"
+  help: "Do you want to define your own data preprocessing function in your algorithm?"
   default: true
 
-partial_function_number_databases:
-  type: int
-  when: "{{ has_partial_function and partial_function_data }}"
-  help: "How many databases do you want to use in your partial function?"
-  default: |-
-    {%- if has_partial_function and partial_function_data -%}
-      1
-    {%- else -%}
-      0
-    {%- endif -%}
-  validator: |-
-    {% if has_partial_function and partial_function_number_databases < 1 %}
-      Must be at least 1
-    {% endif %}
+data_preprocessing_function_name:
+  type: str
+  when: "{{ has_data_preprocessing }}"
+  help: "What is the name of your data preprocessing function?"
+  default: data_preprocessing_function
 
-partial_args:
+data_preprocessing_args:
   type: json
-  when: "{{ has_partial_function }}"
-  help: "Add a list of arguments to the partial function '{{ partial_function_name }}'"
+  when: "{{ has_data_preprocessing }}"
+  help: "Add a list of arguments to the data preprocessing function '{{ data_preprocessing_function_name }}'"
   default: ['arg1', ]
   multiline: true
 
@@ -144,31 +161,6 @@ advanced:
   help: "Do you want to see the advanced options?"
   default: true
 
-# ----------------------- Set up VPN ----------------------- #
-use_vpn:
-  type: bool
-  when: "{{ advanced }}"
-  help: "Do you want to use the VPN network in your algorithm?"
-  default: false
-
-vpn_expose:
-  type: json
-  when: "{{ advanced and use_vpn }}"
-  help: |
-    Configure the ports to expose in the VPN. You can add multiple ports
-    by adding more dictionaries with keys 'label' and 'port'.
-  default: [{'port': 8888, 'label': 'my_label'}]
-  multiline: true
-  validator: |
-    {% for port_dict in vpn_expose %}
-        {% if 'port' not in port_dict %}
-          Each dictionary should have a 'port' key.
-        {% endif %}
-        {% if 'label' not in port_dict %}
-          Each dictionary should have a 'label' key.
-        {% endif %}
-    {% endfor %}
-
 has_gh_pipeline:
   type: bool
   when: "{{ advanced }}"
@@ -219,7 +211,7 @@ _tasks:
   #   {% elif _copier_conf.os == 'windows' %}
   #     Remove-Item LICENSE
   #   {% endif %}
-  # TODO: similarly, delete if empty: central.py, partial.py
+  # TODO: similarly, delete if empty: central.py, federated.py
   # Call python script that cleans up
   - ["{{ _copier_python }}", cleanup.py]
 _message_after_copy: |

macros/kwargs_definition.jinja → macros/arguments_definition.jinja

@@ -1,4 +1,4 @@
-{%- macro kwargs_define(func_name, func_args, num_indents) -%}
+{%- macro arguments_define(func_name, func_args, num_indents) -%}
     {%- if func_args -%}
         # TODO add sensible values{{"\n"}}
         {%- for arg in func_args -%}