From 04f30350abe66504c33db089e525b2a11f09a592 Mon Sep 17 00:00:00 2001 From: Thomas Timmer Date: Wed, 28 Jan 2026 17:38:19 +0100 Subject: [PATCH 01/20] feat: start on implementing the capture api --- helper/data-api/src/context.rs | 118 ++++++++++++++++++ helper/data-api/src/lib.rs | 32 +++-- helper/data-api/wasmcloud.toml | 4 + .../wit/deps/betty-blocks-types/package.wit | 24 ++++ helper/data-api/wit/world.wit | 20 +++ 5 files changed, 187 insertions(+), 11 deletions(-) create mode 100644 helper/data-api/src/context.rs create mode 100644 helper/data-api/wit/deps/betty-blocks-types/package.wit diff --git a/helper/data-api/src/context.rs b/helper/data-api/src/context.rs new file mode 100644 index 0000000..697deae --- /dev/null +++ b/helper/data-api/src/context.rs @@ -0,0 +1,118 @@ +use std::collections::{HashMap, VecDeque}; +use std::sync::{Arc, Mutex}; + +use crate::exports::betty_blocks::data_api::data_api::{ + GuestDataApi, JsonString, +}; +use crate::{inner_request, Config}; + +pub struct DataAPIContext { + application_id: String, + action_id: String, + /// jwt of the customer (so not the jaws jwt used for authenticating the server to server communication) + jwt: Option, + mass_mutate_entries: Arc>>, + capture_stack: Arc>>, +} + +#[derive(Default)] +pub struct MassMutateEntries { + create: VecDeque, + update: VecDeque, + delete: VecDeque, +} + +// replace with real serde_json::Value when implementing +#[allow(non_camel_case_types)] +type serde_json_Value = String; + +pub struct MassMutateEntry { + model_name: String, + variables: serde_json_Value, +} + +impl GuestDataApi for DataAPIContext { + fn new(application_id: String, action_id: String, jwt: Option) -> Self { + DataAPIContext { + application_id, + action_id, + jwt, + mass_mutate_entries: Default::default(), + capture_stack: Default::default(), + } + } + + fn apply_capture(&self) -> Result { + todo!("construct the different mutations and apply them") + } + + fn discard_capture(&self) -> Result { + if let Some(capture_id) = self.capture_stack.lock().expect("lock is poisoned").pop() { + match self + .mass_mutate_entries + .lock() + .expect("lock is poisoned") + .remove(&capture_id) + { + Some(_) => return Ok(format!("deleted capture entry with id {capture_id}")), + None => { + return Ok(format!( + "capture entry with id {capture_id} was already deleted" + )) + } + } + } + + Ok(String::from("nothing to do")) + } + + fn start_capture(&self) -> Result { + let random_capture_id = String::from("hallo1234"); + { + self.mass_mutate_entries + .lock() + .expect("lock is poisoned") + .insert(random_capture_id.clone(), MassMutateEntries::default()); + } + + { + self.capture_stack + .lock() + .expect("lock is poisoned") + .push(random_capture_id.clone()); + } + + Ok(random_capture_id) + } + + fn request(&self, query: String, variables: JsonString) -> Result { + if let Ok(capture_stack) = self.capture_stack.lock() { + if capture_stack.len() > 0 { + todo!("parse gql and put to correct capture stack if it is a single create,update,delete. If it is a many let it continue") + } + } + + let config = match Config::from_env() { + Ok(config) => config, + Err(e) => return Err(format!("Configuration error: {e:#}")), + }; + + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(rt) => rt, + Err(e) => return Err(format!("failed to create tokio runtime: {e}")), + }; + + runtime + .block_on(inner_request( + config, + &self.application_id, + self.jwt.clone(), + query, + variables, + )) + .map_err(|e| format!("{e:#}")) + } +} diff --git a/helper/data-api/src/lib.rs b/helper/data-api/src/lib.rs index 5ffc566..934408f 100644 --- a/helper/data-api/src/lib.rs +++ b/helper/data-api/src/lib.rs @@ -11,7 +11,9 @@ use crate::exports::betty_blocks::data_api::data_api::{Guest, HelperContext}; pub mod data_grpc { tonic::include_proto!("data_grpc"); } +pub mod context; +use crate::context::DataAPIContext; use crate::data_grpc::data_api_client::DataApiClient; use crate::data_grpc::DataApiRequest; use crate::data_grpc::{Context as GrpcContext, DataApiResult}; @@ -21,14 +23,14 @@ wit_bindgen::generate!({ generate_all }); const STATUS_ERROR: i32 = Status::Error as i32; const STATUS_OK: i32 = Status::Ok as i32; -struct Config { +pub struct Config { grpc_server_uri: String, jaws_issuer: String, jaws_secret_key: String, } impl Config { - fn from_env() -> anyhow::Result { + pub fn from_env() -> anyhow::Result { Ok(Self { grpc_server_uri: env::var("GRPC_SERVER_URI") .unwrap_or_else(|_| "http://data-api:50054".to_string()), @@ -38,11 +40,10 @@ impl Config { } } -struct DataApi {} - -export!(DataApi); +pub struct DataApi {} impl Guest for DataApi { + type DataApi = DataAPIContext; fn request( helper_context: HelperContext, query: String, @@ -62,14 +63,23 @@ impl Guest for DataApi { }; runtime - .block_on(inner_request(config, helper_context, query, variables)) + .block_on(inner_request( + config, + &helper_context.application_id, + helper_context.jwt, + query, + variables, + )) .map_err(|e| format!("{e:#}")) } } -async fn inner_request( +export!(DataApi); + +pub async fn inner_request( config: Config, - helper_context: HelperContext, + application_id: &str, + jwt: Option, query: String, variables: String, ) -> anyhow::Result { @@ -87,12 +97,12 @@ async fn inner_request( query, variables, context: Some(GrpcContext { - application_id: helper_context.application_id.to_string(), - jwt: helper_context.jwt.unwrap_or_default(), + application_id: application_id.to_string(), + jwt: jwt.unwrap_or_default(), }), }); - let token = generate_jaws(&config, helper_context.application_id)?; + let token = generate_jaws(&config, application_id.to_string())?; let metadata = request.metadata_mut(); metadata.insert( diff --git a/helper/data-api/wasmcloud.toml b/helper/data-api/wasmcloud.toml index d6f5360..85c33f2 100644 --- a/helper/data-api/wasmcloud.toml +++ b/helper/data-api/wasmcloud.toml @@ -5,3 +5,7 @@ type = "component" [component] wasm_target = "wasm32-wasip2" + +[[registry.pull.sources]] +target = "betty-blocks:types@0.1.0" +source = "file://../http-wrapper/external/actions.wit" diff --git a/helper/data-api/wit/deps/betty-blocks-types/package.wit b/helper/data-api/wit/deps/betty-blocks-types/package.wit new file mode 100644 index 0000000..9ede18e --- /dev/null +++ b/helper/data-api/wit/deps/betty-blocks-types/package.wit @@ -0,0 +1,24 @@ +package betty-blocks:types; + +interface actions { + type json-string = string; + + record payload { + input: json-string, + configurations: json-string, + } + + record input { + action-id: string, + payload: payload, + } + + record output { + %result: json-string, + } + + call: func(input: input) -> result; + + health: func() -> result; +} + diff --git a/helper/data-api/wit/world.wit b/helper/data-api/wit/world.wit index e4dabbd..29fc695 100644 --- a/helper/data-api/wit/world.wit +++ b/helper/data-api/wit/world.wit @@ -1,6 +1,26 @@ package betty-blocks:data-api; interface data-api { + // use types when it exists + use betty-blocks:types/actions.{json-string}; + + resource data-api { + constructor(application-id: string, action-id: string, jwt: option); + + // capture api. Will capture single mutations and when calling apply-capture will apply them all at once. + // the query and many mutations will just pass through it. + + /// start capturing the mutations + start-capture: func() -> result; + /// discard the current captured mutations + discard-capture: func() -> result; + // or flush or commit ? + /// apply all the captured mutations + apply-capture: func() -> result; + + /// do a graphql request to the data-api + request: func(query: string, variables: json-string) -> result; + } record helper-context { application-id: string, From 3fe0dc8e402a401e66ee9df9aa04075198edcaf8 Mon Sep 17 00:00:00 2001 From: Thomas Timmer Date: Wed, 28 Jan 2026 18:20:39 +0100 Subject: [PATCH 02/20] chore: start on graphql parsing --- helper/data-api/src/context.rs | 60 ++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/helper/data-api/src/context.rs b/helper/data-api/src/context.rs index 697deae..bd40bb3 100644 --- a/helper/data-api/src/context.rs +++ b/helper/data-api/src/context.rs @@ -116,3 +116,63 @@ impl GuestDataApi for DataAPIContext { .map_err(|e| format!("{e:#}")) } } + + + +#[test] +fn xd() { + use graphql_parser::{parse_query, minify_query}; + use graphql_parser::query::{ParseError}; + + let query = "mutation {createSong(input: $input)} {id}"; + let xd = parse_query::(query).unwrap(); + let mut mutations = Vec::new(); + + #[derive(Debug)] + enum Blagh { + Create, + Update, + Delete + } + + for def in xd.definitions { + match def { + graphql_parser::query::Definition::Operation(operation) => { + match operation { + graphql_parser::query::OperationDefinition::Mutation(mutation) => { + for item in mutation.selection_set.items { + match item { + graphql_parser::query::Selection::Field(field) if field.name.starts_with("create") && !field.name.starts_with("createMany") => { + let name = field.name.split_at("create".len()).1.to_string(); + mutations.push((Blagh::Create, name)); + } + graphql_parser::query::Selection::Field(field) if field.name.starts_with("update") && !field.name.starts_with("updateMany") => { + let name = field.name.split_at("update".len()).1.to_string(); + mutations.push((Blagh::Update, name)); + } + graphql_parser::query::Selection::Field(field) if field.name.starts_with("delete") && !field.name.starts_with("deleteMany") => { + let name = field.name.split_at("delete".len()).1.to_string(); + mutations.push((Blagh::Delete, name)); + } + _ => unimplemented!("other than field") + } + } + } + graphql_parser::query::OperationDefinition::SelectionSet(set) => { + for item in set.items { + match item { + graphql_parser::query::Selection::Field(field) if field.name == "id" => { + } + _ => unimplemented!("other than id") + } + } + } + _ => unimplemented!("other than mutation") + } + } + _ => unimplemented!("other than operation") + } + } + dbg!(mutations); + panic!() +} From dccada1b770adf609a017fda10497e288eb37f82 Mon Sep 17 00:00:00 2001 From: Thomas Timmer Date: Thu, 29 Jan 2026 18:46:08 +0100 Subject: [PATCH 03/20] feat: implement the graphql parsing and extracting data from the mutation --- helper/data-api/Cargo.lock | 21 +++ helper/data-api/Cargo.toml | 2 + helper/data-api/src/context.rs | 279 +++++++++++++++++++++++++++------ 3 files changed, 251 insertions(+), 51 deletions(-) diff --git a/helper/data-api/Cargo.lock b/helper/data-api/Cargo.lock index 417840a..7b649a5 100644 --- a/helper/data-api/Cargo.lock +++ b/helper/data-api/Cargo.lock @@ -429,6 +429,16 @@ dependencies = [ "thiserror 2.0.17", ] +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -704,6 +714,7 @@ version = "0.1.0" dependencies = [ "anyhow", "bytes", + "graphql-parser", "jaws-rs", "prost 0.13.5", "reqwest", @@ -1128,6 +1139,16 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "graphql-parser" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a818c0d883d7c0801df27be910917750932be279c7bc82dc541b8769425f409" +dependencies = [ + "combine", + "thiserror 1.0.69", +] + [[package]] name = "h2" version = "0.4.12" diff --git a/helper/data-api/Cargo.toml b/helper/data-api/Cargo.toml index 0d92b29..13c5bfa 100644 --- a/helper/data-api/Cargo.toml +++ b/helper/data-api/Cargo.toml @@ -18,6 +18,8 @@ jaws-rs = { git = "https://github.com/bettyblocks/jaws-rs.git", version = "0.1.0 tonic = { version = "0.13", default-features = false, features = ["codegen", "prost"] } uuid = { version = "1.8.0", features = ["v4"] } tracing = "0.1" +graphql-parser = "0.4.1" +serde_json = "1.0.145" [dev-dependencies] wash-runtime = { git = "https://github.com/bettyblocks/wash", features = ["grpc"], branch = "feat/add-grpc-client" } diff --git a/helper/data-api/src/context.rs b/helper/data-api/src/context.rs index bd40bb3..38ee6a5 100644 --- a/helper/data-api/src/context.rs +++ b/helper/data-api/src/context.rs @@ -1,9 +1,7 @@ use std::collections::{HashMap, VecDeque}; use std::sync::{Arc, Mutex}; -use crate::exports::betty_blocks::data_api::data_api::{ - GuestDataApi, JsonString, -}; +use crate::exports::betty_blocks::data_api::data_api::{GuestDataApi, JsonString}; use crate::{inner_request, Config}; pub struct DataAPIContext { @@ -22,13 +20,9 @@ pub struct MassMutateEntries { delete: VecDeque, } -// replace with real serde_json::Value when implementing -#[allow(non_camel_case_types)] -type serde_json_Value = String; - pub struct MassMutateEntry { model_name: String, - variables: serde_json_Value, + variables: serde_json::Map, } impl GuestDataApi for DataAPIContext { @@ -117,62 +111,245 @@ impl GuestDataApi for DataAPIContext { } } +#[derive(Debug, PartialEq)] +pub enum DataMutation { + Create, + Update, + Delete, +} +#[derive(Debug, PartialEq)] +pub struct Intruction { + operator: DataMutation, + name: String, + data: Option>, +} -#[test] -fn xd() { - use graphql_parser::{parse_query, minify_query}; - use graphql_parser::query::{ParseError}; +impl Intruction { + pub fn create(name: String) -> Self { + Intruction { + name, + operator: DataMutation::Create, + data: None, + } + } - let query = "mutation {createSong(input: $input)} {id}"; - let xd = parse_query::(query).unwrap(); - let mut mutations = Vec::new(); - - #[derive(Debug)] - enum Blagh { - Create, - Update, - Delete + pub fn update(name: String) -> Self { + Intruction { + name, + operator: DataMutation::Update, + data: None, + } } - for def in xd.definitions { + pub fn delete(name: String) -> Self { + Intruction { + name, + operator: DataMutation::Delete, + data: None, + } + } +} + +use graphql_parser::parse_query; +use graphql_parser::query::ParseError; +use serde_json::Number; + +pub fn parse_graphql_to_intruction(graphql: &str) -> Result, ParseError> { + let document = parse_query::(graphql)?; + let mut instruction = None; + + // TODO: get the inlined data from the mutation + for def in document.definitions { match def { - graphql_parser::query::Definition::Operation(operation) => { - match operation { - graphql_parser::query::OperationDefinition::Mutation(mutation) => { - for item in mutation.selection_set.items { - match item { - graphql_parser::query::Selection::Field(field) if field.name.starts_with("create") && !field.name.starts_with("createMany") => { - let name = field.name.split_at("create".len()).1.to_string(); - mutations.push((Blagh::Create, name)); - } - graphql_parser::query::Selection::Field(field) if field.name.starts_with("update") && !field.name.starts_with("updateMany") => { - let name = field.name.split_at("update".len()).1.to_string(); - mutations.push((Blagh::Update, name)); - } - graphql_parser::query::Selection::Field(field) if field.name.starts_with("delete") && !field.name.starts_with("deleteMany") => { - let name = field.name.split_at("delete".len()).1.to_string(); - mutations.push((Blagh::Delete, name)); - } - _ => unimplemented!("other than field") + graphql_parser::query::Definition::Operation(operation) => match operation { + graphql_parser::query::OperationDefinition::Mutation(mutation) => { + if mutation.selection_set.items.len() > 1 { + return Ok(None); + } + + for item in mutation.selection_set.items { + match item { + graphql_parser::query::Selection::Field(field) + if field.name.starts_with("create") + && !field.name.starts_with("createMany") => + { + let name = field.name.split_at("create".len()).1.to_string(); + let mut new_instruction = Intruction::create(name); + + extract_values_from_gql_argument( + &mut new_instruction, + field.arguments, + ); + + instruction = Some(new_instruction); + } + graphql_parser::query::Selection::Field(field) + if field.name.starts_with("update") + && !field.name.starts_with("updateMany") => + { + let name = field.name.split_at("update".len()).1.to_string(); + let mut new_instruction = Intruction::update(name); + + extract_values_from_gql_argument( + &mut new_instruction, + field.arguments, + ); + + instruction = Some(new_instruction); } + graphql_parser::query::Selection::Field(field) + if field.name.starts_with("delete") + && !field.name.starts_with("deleteMany") => + { + let name = field.name.split_at("delete".len()).1.to_string(); + let mut new_instruction = Intruction::delete(name); + + extract_values_from_gql_argument( + &mut new_instruction, + field.arguments, + ); + + instruction = Some(new_instruction); + } + _ => {} } } - graphql_parser::query::OperationDefinition::SelectionSet(set) => { - for item in set.items { - match item { - graphql_parser::query::Selection::Field(field) if field.name == "id" => { - } - _ => unimplemented!("other than id") - } + } + graphql_parser::query::OperationDefinition::SelectionSet(set) => { + for item in set.items { + match item { + graphql_parser::query::Selection::Field(field) + if field.name == "id" => {} + _ => {} } } - _ => unimplemented!("other than mutation") + } + _ => {} + }, + _ => {} + } + } + + Ok(instruction) +} + +fn extract_values_from_gql_argument( + instruction: &mut Intruction, + arguments: Vec<(String, graphql_parser::query::Value<'_, String>)>, +) { + let mut tmp_data = serde_json::Map::new(); + for (key, value) in arguments { + if let Some(json) = graphql_to_json(value) { + tmp_data.insert(key, json); + } + } + if !tmp_data.is_empty() { + instruction.data = Some(tmp_data); + } +} + +fn graphql_to_json(val: graphql_parser::query::Value<'_, String>) -> Option { + match val { + graphql_parser::query::Value::Variable(_) => return None, + graphql_parser::query::Value::Boolean(b) => return Some(serde_json::Value::Bool(b)), + graphql_parser::query::Value::String(s) => { + return Some(serde_json::Value::String(s)); + } + graphql_parser::query::Value::Int(i) => { + let num = i.as_i64()?; + return Some(serde_json::Value::Number(num.into())); + } + graphql_parser::query::Value::Float(f) => { + return Some(serde_json::Value::Number(Number::from_f64(f)?)); + } + graphql_parser::query::Value::Null => { + return Some(serde_json::Value::Null); + } + graphql_parser::query::Value::Enum(s) => { + return Some(serde_json::Value::String(s)); + } + graphql_parser::query::Value::List(l) => { + let list = l.into_iter().flat_map(graphql_to_json).collect(); + return Some(serde_json::Value::Array(list)); + } + graphql_parser::query::Value::Object(m) => { + let mut map = serde_json::Map::new(); + for (k, v) in m { + if let Some(json) = graphql_to_json(v) { + map.insert(k, json); } } - _ => unimplemented!("other than operation") + return Some(serde_json::Value::Object(map)); } } - dbg!(mutations); - panic!() +} + +#[test] +fn parse_graphql_to_intruction_test() { + let query = "mutation {createSong(input: $input)} {id}"; + let out = parse_graphql_to_intruction(query).unwrap().unwrap(); + assert_eq!(out, Intruction::create("Song".to_string())); + + let query = r#" + mutation ($input: userInput, $validationSets: [String]) { + createuser(input: $input, validationSets: $validationSets) { + id + } + }"#; + + let out = parse_graphql_to_intruction(query).unwrap().unwrap(); + assert_eq!(out, Intruction::create("user".to_string())); + + let query = r#" + mutation ($id: Int!, $input: userInput, $validationSets: [String]) { + updateuser(id: $id, input: $input, validationSets: $validationSets) { + id + } + }"#; + + let out = parse_graphql_to_intruction(query).unwrap().unwrap(); + assert_eq!(out, Intruction::update("user".to_string())); + + let query = r#" + mutation ($id: Int!) { + deleteuser(id: $id) { + id + } + }"#; + + let out = parse_graphql_to_intruction(query).unwrap().unwrap(); + assert_eq!(out, Intruction::delete("user".to_string())); + + let query = r#" + { + allSong { + results { + id + name + } + } + }"#; + + let out = parse_graphql_to_intruction(query).unwrap(); + assert_eq!(out, None); +} + +#[test] +fn extract_data_from_mutation() { + let query = r#" + mutation { + deleteuser(id: 2) { + id + } + }"#; + + let out = parse_graphql_to_intruction(query).unwrap().unwrap(); + let mut instruction = Intruction::delete("user".to_string()); + instruction.data = Some(serde_json::Map::from_iter([( + String::from("id"), + serde_json::Value::Number(2.into()), + )])); + + assert_eq!(out, instruction); } From 33fbb9adc05d386ab9e6f0e615a1f0136bdbf2ba Mon Sep 17 00:00:00 2001 From: Thomas Timmer Date: Mon, 9 Mar 2026 11:41:40 +0100 Subject: [PATCH 04/20] fix: get data from gql mutation --- helper/data-api/src/context.rs | 73 +++++++++++++++++++++++++++++++++- helper/data-api/wit/world.wit | 13 ++++++ 2 files changed, 84 insertions(+), 2 deletions(-) diff --git a/helper/data-api/src/context.rs b/helper/data-api/src/context.rs index 38ee6a5..df078dc 100644 --- a/helper/data-api/src/context.rs +++ b/helper/data-api/src/context.rs @@ -1,7 +1,7 @@ use std::collections::{HashMap, VecDeque}; use std::sync::{Arc, Mutex}; -use crate::exports::betty_blocks::data_api::data_api::{GuestDataApi, JsonString}; +use crate::exports::betty_blocks::data_api::data_api::{self, GuestDataApi, JsonString}; use crate::{inner_request, Config}; pub struct DataAPIContext { @@ -20,11 +20,51 @@ pub struct MassMutateEntries { delete: VecDeque, } +impl MassMutateEntries { + fn as_pending_mutations(&self) -> Vec> { + vec![ + self.create + .iter() + .map(|x| x.as_pending_mutation(DataMutation::Create)) + .collect(), + self.update + .iter() + .map(|x| x.as_pending_mutation(DataMutation::Update)) + .collect(), + self.delete + .iter() + .map(|x| x.as_pending_mutation(DataMutation::Delete)) + .collect(), + ] + } +} + pub struct MassMutateEntry { model_name: String, variables: serde_json::Map, } +impl MassMutateEntry { + fn as_pending_mutation(&self, operation: DataMutation) -> data_api::PendingMutation { + data_api::PendingMutation { + mutation_name: format_mutation_name(&self.model_name, &operation), + mutation: format_mutation(&self.model_name, &operation), + variables: serde_json::to_string(&self.variables).expect("incorrect variables"), + } + } +} + +fn format_mutation(model_name: &str, operation: &DataMutation) -> String { + format!( + "mutation {{ {}{model_name}(input: $input) {{ id }} }}", + operation.as_static_str() + ) +} + +fn format_mutation_name(model_name: &str, operation: &DataMutation) -> String { + format!("{}{model_name}", operation.as_static_str()) +} + impl GuestDataApi for DataAPIContext { fn new(application_id: String, action_id: String, jwt: Option) -> Self { DataAPIContext { @@ -79,6 +119,20 @@ impl GuestDataApi for DataAPIContext { Ok(random_capture_id) } + fn pending_capture(&self) -> Result>, String> { + if let Some(capture_id) = self.capture_stack.lock().expect("lock is poisoned").last() { + let lock = self.mass_mutate_entries.lock().expect("lock is poisoned"); + + let entries = lock + .get(capture_id) + .expect("capture stack and mass mutate entries out of sync"); + + return Ok(entries.as_pending_mutations()); + } + + Ok(Vec::new()) + } + fn request(&self, query: String, variables: JsonString) -> Result { if let Ok(capture_stack) = self.capture_stack.lock() { if capture_stack.len() > 0 { @@ -86,6 +140,10 @@ impl GuestDataApi for DataAPIContext { } } + self.request_raw(query, variables) + } + + fn request_raw(&self, query: String, variables: JsonString) -> Result { let config = match Config::from_env() { Ok(config) => config, Err(e) => return Err(format!("Configuration error: {e:#}")), @@ -118,6 +176,18 @@ pub enum DataMutation { Delete, } +impl DataMutation { + fn as_static_str(&self) -> &'static str { + use DataMutation::*; + + match self { + Create => "create", + Update => "update", + Delete => "delete", + } + } +} + #[derive(Debug, PartialEq)] pub struct Intruction { operator: DataMutation, @@ -159,7 +229,6 @@ pub fn parse_graphql_to_intruction(graphql: &str) -> Result, let document = parse_query::(graphql)?; let mut instruction = None; - // TODO: get the inlined data from the mutation for def in document.definitions { match def { graphql_parser::query::Definition::Operation(operation) => match operation { diff --git a/helper/data-api/wit/world.wit b/helper/data-api/wit/world.wit index 29fc695..b431744 100644 --- a/helper/data-api/wit/world.wit +++ b/helper/data-api/wit/world.wit @@ -4,9 +4,16 @@ interface data-api { // use types when it exists use betty-blocks:types/actions.{json-string}; + record pending-mutation { + mutation-name: string, + mutation: string, + variables: json-string, + } + resource data-api { constructor(application-id: string, action-id: string, jwt: option); + // capture api. Will capture single mutations and when calling apply-capture will apply them all at once. // the query and many mutations will just pass through it. @@ -18,8 +25,14 @@ interface data-api { /// apply all the captured mutations apply-capture: func() -> result; + /// returns the current pending capture, grouped by the order how they need to be applied + pending-capture: func() -> result>, string>; + /// do a graphql request to the data-api request: func(query: string, variables: json-string) -> result; + + /// do a graphql request to the data-api, only use when you know what you are doing + request-raw: func(query: string, variables: json-string) -> result; } record helper-context { From 75a63d82ab295458694537efaf57eb8359261efa Mon Sep 17 00:00:00 2001 From: Robin Walthuis Date: Mon, 1 Jun 2026 08:41:31 +0200 Subject: [PATCH 05/20] feat: add partial capture application and request parsing logic --- helper/data-api/Cargo.toml | 21 +++- helper/data-api/src/context.rs | 216 +++++++++++++++++++++++---------- helper/data-api/wit/world.wit | 2 +- 3 files changed, 170 insertions(+), 69 deletions(-) diff --git a/helper/data-api/Cargo.toml b/helper/data-api/Cargo.toml index 13c5bfa..c8dee32 100644 --- a/helper/data-api/Cargo.toml +++ b/helper/data-api/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "data-api-component" version = "0.1.0" -edition = "2021" +edition = "2024" publish = false [lib] @@ -10,24 +10,33 @@ crate-type = ["cdylib"] [dependencies] anyhow = "1.0.100" wit-bindgen = "0.48.0" -wasmcloud-grpc-client = { git = "https://github.com/Aditya1404Sal/wasmcloud-grpc-client.git", branch="master" } +wasmcloud-grpc-client = { git = "https://github.com/Aditya1404Sal/wasmcloud-grpc-client.git", branch = "master" } wasmcloud-component = "0.2.0" tokio = { version = "1.48.0", features = ["rt"] } prost = "0.13" jaws-rs = { git = "https://github.com/bettyblocks/jaws-rs.git", version = "0.1.0" } -tonic = { version = "0.13", default-features = false, features = ["codegen", "prost"] } +tonic = { version = "0.13", default-features = false, features = [ + "codegen", + "prost", +] } uuid = { version = "1.8.0", features = ["v4"] } tracing = "0.1" graphql-parser = "0.4.1" serde_json = "1.0.145" [dev-dependencies] -wash-runtime = { git = "https://github.com/bettyblocks/wash", features = ["grpc"], branch = "feat/add-grpc-client" } -tonic = { version = "0.13", default-features = false, features = ["codegen", "prost", "transport"] } +wash-runtime = { git = "https://github.com/bettyblocks/wash", features = [ + "grpc", +], branch = "feat/add-grpc-client" } +tonic = { version = "0.13", default-features = false, features = [ + "codegen", + "prost", + "transport", +] } serde_json = "1.0.145" bytes = "1.11.0" semver = "1.0.27" -reqwest = {version = "0.12.24", default-features = false, features = ["json"] } +reqwest = { version = "0.12.24", default-features = false, features = ["json"] } tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] } serial_test = "3.2.0" diff --git a/helper/data-api/src/context.rs b/helper/data-api/src/context.rs index df078dc..84d84d8 100644 --- a/helper/data-api/src/context.rs +++ b/helper/data-api/src/context.rs @@ -6,23 +6,26 @@ use crate::{inner_request, Config}; pub struct DataAPIContext { application_id: String, + + // TODO: Check if this is still dead + #[allow(dead_code)] action_id: String, /// jwt of the customer (so not the jaws jwt used for authenticating the server to server communication) jwt: Option, - mass_mutate_entries: Arc>>, - capture_stack: Arc>>, + capture_stack: Arc>>, } -#[derive(Default)] +#[derive(Default, Debug)] pub struct MassMutateEntries { create: VecDeque, update: VecDeque, + upsert: VecDeque, delete: VecDeque, } impl MassMutateEntries { - fn as_pending_mutations(&self) -> Vec> { - vec![ + fn as_pending_mutations(&self) -> [Vec; 4] { + [ self.create .iter() .map(|x| x.as_pending_mutation(DataMutation::Create)) @@ -31,6 +34,10 @@ impl MassMutateEntries { .iter() .map(|x| x.as_pending_mutation(DataMutation::Update)) .collect(), + self.upsert + .iter() + .map(|x| x.as_pending_mutation(DataMutation::Upsert)) + .collect(), self.delete .iter() .map(|x| x.as_pending_mutation(DataMutation::Delete)) @@ -39,6 +46,7 @@ impl MassMutateEntries { } } +#[derive(Debug)] pub struct MassMutateEntry { model_name: String, variables: serde_json::Map, @@ -65,78 +73,129 @@ fn format_mutation_name(model_name: &str, operation: &DataMutation) -> String { format!("{}{model_name}", operation.as_static_str()) } +fn generate_delayed_id(id: &str, mutation_name: &str) -> JsonString { + format!(r#"{{"data": {{"{mutation_name}": {{"id": "{id}"}}}}}}"#) +} + impl GuestDataApi for DataAPIContext { fn new(application_id: String, action_id: String, jwt: Option) -> Self { DataAPIContext { application_id, action_id, jwt, - mass_mutate_entries: Default::default(), capture_stack: Default::default(), } } fn apply_capture(&self) -> Result { - todo!("construct the different mutations and apply them") - } + if let Some(MassMutateEntries { create, update, upsert, delete }) = self.capture_stack.lock().expect("lock is poisoned").pop() { + /* TODO: resolve ids + Locally generated ids need to be distinguishable from normal ids after being sent back to the caller and then sent here as for example a related row. + This will probably be done with negative numbered ids? Unless we have a better way of distinguising them. + Preferrably the caller wouldn't be able to distinguish them. + + Here, we need to go through all reserved ids. + In theory the relations would still work if we only reserve ids that are referenced as relations by other mutations, and set the rest to default. + However, that would mess with the order of the inserted records. + I believe it would be better to preserve the insert order of the action, but it is an optimization worth considering. + These need to be reserved by a query, and we need to map them to the locally generated ids. + Then we need to replace all locally generated ids with the real ids. + Then we can upsert all creates, upsert all updates, upsert all upserts and delete all deletes. + These can be clustered by model, so we can do upsertManyUser but not upsertManyUserAndRole. + */ + + // If we want to be able to specify a unique by for the upsert we would want to handle them separately, + // but this is left out of cope for now. + let upsert_manys = [create, update, upsert].into_iter().flatten().fold(HashMap::new(), |mut map, item| {map.entry(item.model_name).or_insert(Vec::new()).push(item.variables); map}); + + // TODO: Do we want to delete entries before they get upserted where possible? + // Would mean less unnecessary mutations in the data api, but likely worse performance if we need to check for that here. + // It would look something like this. + // for entry in delete { + // if let Some(id) = entry.variables.get("id") && let Some(model_entry) = x.get_mut(&entry.model_name) { + // model_entry.retain(|v| !v.get("id").is_some_and(|y| y == id)) + // // Still need to delete if this doesn't match anything + // } + // } + + for (model_name, variables) in upsert_manys { + let query = format!( + "mutation {{ upsertMany{model_name}(input: $input) {{ id }} }}" + ); + + let variables = format!("{{\"input\": {}}}", serde_json::to_string(&variables).map_err(|_| String::from("could not format input variables"))?); + + // TODO: set up some kind of mocking so this doesn't break when testing. Same goes for the delete manys. + // self.request_raw(query, variables)?; + } - fn discard_capture(&self) -> Result { - if let Some(capture_id) = self.capture_stack.lock().expect("lock is poisoned").pop() { - match self - .mass_mutate_entries - .lock() - .expect("lock is poisoned") - .remove(&capture_id) - { - Some(_) => return Ok(format!("deleted capture entry with id {capture_id}")), - None => { - return Ok(format!( - "capture entry with id {capture_id} was already deleted" - )) - } + let delete_manys = delete.into_iter().fold(HashMap::new(), |mut map, mut item| {if let Some(id) = item.variables.remove("id") {map.entry(item.model_name).or_insert(Vec::new()).push(id);} map}); + + for (model_name, variables) in delete_manys { + let query = format!( + "mutation {{ deleteMany{model_name}(input: $input) {{ id }} }}" + ); + + let variables = format!("{{\"input\": {{\"ids\": {}}}}}", serde_json::to_string(&variables).map_err(|_| String::from("could not format input variables"))?); + + // self.request_raw(query, variables)?; } } Ok(String::from("nothing to do")) } - fn start_capture(&self) -> Result { - let random_capture_id = String::from("hallo1234"); - { - self.mass_mutate_entries - .lock() - .expect("lock is poisoned") - .insert(random_capture_id.clone(), MassMutateEntries::default()); + fn discard_capture(&self) -> Result { + if let Some(_) = self.capture_stack.lock().expect("lock is poisoned").pop() { + return Ok(format!("deleted most recent capture stack entry")); } - { - self.capture_stack - .lock() - .expect("lock is poisoned") - .push(random_capture_id.clone()); - } + Ok(String::from("nothing to do")) + } - Ok(random_capture_id) + fn start_capture(&self) -> Result<(), String> { + self.capture_stack + .lock() + .map_err(|_| String::from("capture stack lock poisoned"))? + .push(MassMutateEntries::default()); + Ok(()) } fn pending_capture(&self) -> Result>, String> { - if let Some(capture_id) = self.capture_stack.lock().expect("lock is poisoned").last() { - let lock = self.mass_mutate_entries.lock().expect("lock is poisoned"); - - let entries = lock - .get(capture_id) - .expect("capture stack and mass mutate entries out of sync"); - - return Ok(entries.as_pending_mutations()); + if let Some(entries) = self.capture_stack.lock().expect("lock is poisoned").last() { + return Ok(entries.as_pending_mutations().to_vec()); } Ok(Vec::new()) } fn request(&self, query: String, variables: JsonString) -> Result { - if let Ok(capture_stack) = self.capture_stack.lock() { - if capture_stack.len() > 0 { - todo!("parse gql and put to correct capture stack if it is a single create,update,delete. If it is a many let it continue") + if let Ok(mut capture_stack) = self.capture_stack.lock() { + if let Some(entries) = capture_stack.last_mut() { + let query_remainder = match query.split_once("mutation") { + Some((_, remainder)) => remainder, + None => return self.request_raw(query, variables) + }; + + let query_remainder = query_remainder.split_once('{').ok_or_else(|| String::from("query is improperly formatted"))?.1.trim(); + + if query_remainder[6..10] == *"Many" { + return self.request_raw(query, variables) + } + + let mutation_name = query_remainder.split_once(|c: char| c.is_whitespace() || c == '(').ok_or_else(|| String::from("query is improperly formatted"))?.0; + + match &query_remainder[0..6] { + "create" => &mut entries.create, + "update" => &mut entries.update, + "upsert" => &mut entries.upsert, + "delete" => &mut entries.delete, + _ => return self.request_raw(query, variables) + }.push_back(MassMutateEntry { model_name: mutation_name[6..].to_string(), variables: serde_json::from_str(&variables).map_err(|_| String::from("could not parse variables"))? }); + + // TODO: Generate a local id that will be resolved to a real id later. + // Read more about this logic in [[Self::apply_capture]] + return Ok(generate_delayed_id("1", mutation_name)); } } @@ -173,6 +232,7 @@ impl GuestDataApi for DataAPIContext { pub enum DataMutation { Create, Update, + Upsert, Delete, } @@ -183,21 +243,22 @@ impl DataMutation { match self { Create => "create", Update => "update", + Upsert => "upsert", Delete => "delete", } } } #[derive(Debug, PartialEq)] -pub struct Intruction { +pub struct Instruction { operator: DataMutation, name: String, data: Option>, } -impl Intruction { +impl Instruction { pub fn create(name: String) -> Self { - Intruction { + Instruction { name, operator: DataMutation::Create, data: None, @@ -205,15 +266,23 @@ impl Intruction { } pub fn update(name: String) -> Self { - Intruction { + Instruction { name, operator: DataMutation::Update, data: None, } } + pub fn upsert(name: String) -> Self { + Instruction { + name, + operator: DataMutation::Upsert, + data: None, + } + } + pub fn delete(name: String) -> Self { - Intruction { + Instruction { name, operator: DataMutation::Delete, data: None, @@ -225,7 +294,7 @@ use graphql_parser::parse_query; use graphql_parser::query::ParseError; use serde_json::Number; -pub fn parse_graphql_to_intruction(graphql: &str) -> Result, ParseError> { +pub fn parse_graphql_to_intruction(graphql: &str) -> Result, ParseError> { let document = parse_query::(graphql)?; let mut instruction = None; @@ -244,7 +313,7 @@ pub fn parse_graphql_to_intruction(graphql: &str) -> Result, && !field.name.starts_with("createMany") => { let name = field.name.split_at("create".len()).1.to_string(); - let mut new_instruction = Intruction::create(name); + let mut new_instruction = Instruction::create(name); extract_values_from_gql_argument( &mut new_instruction, @@ -258,7 +327,7 @@ pub fn parse_graphql_to_intruction(graphql: &str) -> Result, && !field.name.starts_with("updateMany") => { let name = field.name.split_at("update".len()).1.to_string(); - let mut new_instruction = Intruction::update(name); + let mut new_instruction = Instruction::update(name); extract_values_from_gql_argument( &mut new_instruction, @@ -272,7 +341,7 @@ pub fn parse_graphql_to_intruction(graphql: &str) -> Result, && !field.name.starts_with("deleteMany") => { let name = field.name.split_at("delete".len()).1.to_string(); - let mut new_instruction = Intruction::delete(name); + let mut new_instruction = Instruction::delete(name); extract_values_from_gql_argument( &mut new_instruction, @@ -304,7 +373,7 @@ pub fn parse_graphql_to_intruction(graphql: &str) -> Result, } fn extract_values_from_gql_argument( - instruction: &mut Intruction, + instruction: &mut Instruction, arguments: Vec<(String, graphql_parser::query::Value<'_, String>)>, ) { let mut tmp_data = serde_json::Map::new(); @@ -354,11 +423,34 @@ fn graphql_to_json(val: graphql_parser::query::Value<'_, String>) -> Option result; + start-capture: func() -> result<_, string>; /// discard the current captured mutations discard-capture: func() -> result; // or flush or commit ? From 869d47c65bc1987886ba419486ac84a085f57fcf Mon Sep 17 00:00:00 2001 From: Dewinz Date: Wed, 3 Jun 2026 16:01:32 +0200 Subject: [PATCH 06/20] feat: count the amount of negative ids in upsert_manys for mass mutate apply_capture --- helper/data-api/src/context.rs | 172 +++++++++++++++--- helper/data-api/src/lib.rs | 2 +- .../tests/data_api_integration_test.rs | 6 +- 3 files changed, 148 insertions(+), 32 deletions(-) diff --git a/helper/data-api/src/context.rs b/helper/data-api/src/context.rs index 84d84d8..4b29c91 100644 --- a/helper/data-api/src/context.rs +++ b/helper/data-api/src/context.rs @@ -1,8 +1,9 @@ use std::collections::{HashMap, VecDeque}; +use std::default; use std::sync::{Arc, Mutex}; use crate::exports::betty_blocks::data_api::data_api::{self, GuestDataApi, JsonString}; -use crate::{inner_request, Config}; +use crate::{Config, inner_request}; pub struct DataAPIContext { application_id: String, @@ -88,7 +89,13 @@ impl GuestDataApi for DataAPIContext { } fn apply_capture(&self) -> Result { - if let Some(MassMutateEntries { create, update, upsert, delete }) = self.capture_stack.lock().expect("lock is poisoned").pop() { + if let Some(MassMutateEntries { + create, + update, + upsert, + delete, + }) = self.capture_stack.lock().expect("lock is poisoned").pop() + { /* TODO: resolve ids Locally generated ids need to be distinguishable from normal ids after being sent back to the caller and then sent here as for example a related row. This will probably be done with negative numbered ids? Unless we have a better way of distinguising them. @@ -106,37 +113,61 @@ impl GuestDataApi for DataAPIContext { // If we want to be able to specify a unique by for the upsert we would want to handle them separately, // but this is left out of cope for now. - let upsert_manys = [create, update, upsert].into_iter().flatten().fold(HashMap::new(), |mut map, item| {map.entry(item.model_name).or_insert(Vec::new()).push(item.variables); map}); - + let upsert_manys = [create, update, upsert].into_iter().flatten().fold( + HashMap::new(), + |mut map, item| { + map.entry(item.model_name) + .or_insert(Vec::new()) + .push(item.variables); + map + }, + ); + // TODO: Do we want to delete entries before they get upserted where possible? // Would mean less unnecessary mutations in the data api, but likely worse performance if we need to check for that here. // It would look something like this. // for entry in delete { // if let Some(id) = entry.variables.get("id") && let Some(model_entry) = x.get_mut(&entry.model_name) { - // model_entry.retain(|v| !v.get("id").is_some_and(|y| y == id)) + // model_entry.retain(|v| !v.get("id").is_some_and(|y| y == id)) // // Still need to delete if this doesn't match anything // } // } - for (model_name, variables) in upsert_manys { - let query = format!( - "mutation {{ upsertMany{model_name}(input: $input) {{ id }} }}" - ); + for (model_name, all_variables) in upsert_manys { + let _model_and_amount_of_negative_ids = + count_negative_ids_in_maps(&model_name, &all_variables); - let variables = format!("{{\"input\": {}}}", serde_json::to_string(&variables).map_err(|_| String::from("could not format input variables"))?); + let query = + format!("mutation {{ upsertMany{model_name}(input: $input) {{ id }} }}"); + + let variables = format!( + "{{\"input\": {}}}", + serde_json::to_string(&all_variables) + .map_err(|_| String::from("could not format input variables"))? + ); // TODO: set up some kind of mocking so this doesn't break when testing. Same goes for the delete manys. // self.request_raw(query, variables)?; } - let delete_manys = delete.into_iter().fold(HashMap::new(), |mut map, mut item| {if let Some(id) = item.variables.remove("id") {map.entry(item.model_name).or_insert(Vec::new()).push(id);} map}); + let delete_manys = delete + .into_iter() + .fold(HashMap::new(), |mut map, mut item| { + if let Some(id) = item.variables.remove("id") { + map.entry(item.model_name).or_insert(Vec::new()).push(id); + } + map + }); for (model_name, variables) in delete_manys { - let query = format!( - "mutation {{ deleteMany{model_name}(input: $input) {{ id }} }}" - ); + let query = + format!("mutation {{ deleteMany{model_name}(input: $input) {{ id }} }}"); - let variables = format!("{{\"input\": {{\"ids\": {}}}}}", serde_json::to_string(&variables).map_err(|_| String::from("could not format input variables"))?); + let variables = format!( + "{{\"input\": {{\"ids\": {}}}}}", + serde_json::to_string(&variables) + .map_err(|_| String::from("could not format input variables"))? + ); // self.request_raw(query, variables)?; } @@ -174,26 +205,38 @@ impl GuestDataApi for DataAPIContext { if let Some(entries) = capture_stack.last_mut() { let query_remainder = match query.split_once("mutation") { Some((_, remainder)) => remainder, - None => return self.request_raw(query, variables) + None => return self.request_raw(query, variables), }; - let query_remainder = query_remainder.split_once('{').ok_or_else(|| String::from("query is improperly formatted"))?.1.trim(); + let query_remainder = query_remainder + .split_once('{') + .ok_or_else(|| String::from("query is improperly formatted"))? + .1 + .trim(); if query_remainder[6..10] == *"Many" { - return self.request_raw(query, variables) + return self.request_raw(query, variables); } - let mutation_name = query_remainder.split_once(|c: char| c.is_whitespace() || c == '(').ok_or_else(|| String::from("query is improperly formatted"))?.0; + let mutation_name = query_remainder + .split_once(|c: char| c.is_whitespace() || c == '(') + .ok_or_else(|| String::from("query is improperly formatted"))? + .0; match &query_remainder[0..6] { "create" => &mut entries.create, "update" => &mut entries.update, "upsert" => &mut entries.upsert, "delete" => &mut entries.delete, - _ => return self.request_raw(query, variables) - }.push_back(MassMutateEntry { model_name: mutation_name[6..].to_string(), variables: serde_json::from_str(&variables).map_err(|_| String::from("could not parse variables"))? }); - - // TODO: Generate a local id that will be resolved to a real id later. + _ => return self.request_raw(query, variables), + } + .push_back(MassMutateEntry { + model_name: mutation_name[6..].to_string(), + variables: serde_json::from_str(&variables) + .map_err(|_| String::from("could not parse variables"))?, + }); + + // TODO: Generate a local id that will be resolved to a real id later. // Read more about this logic in [[Self::apply_capture]] return Ok(generate_delayed_id("1", mutation_name)); } @@ -228,6 +271,59 @@ impl GuestDataApi for DataAPIContext { } } +/// Loops through a vec of serde_json::Maps and counts the amount of negative ids along with which +/// model name they correspond to. +fn count_negative_ids_in_maps( + current_model: &str, + all_variables: &Vec>, +) -> HashMap { + let mut map: HashMap = HashMap::new(); + + for variables in all_variables { + count_negative_ids_in_map(&mut map, current_model, variables); + } + + map +} + +/// Loops through the entries of a serde_json::Map and counts the amount of negative ids along with +/// which model name they correspond to and puts it into the given HashMap. +fn count_negative_ids_in_map( + map: &mut HashMap, + current_model: &str, + variables: &serde_json::Map, +) { + for (key, value) in variables { + if key == "id" { + if let Some(id) = value.as_i64() + && id < 0 + { + increment_mapped_value_by(map, String::from(current_model), 1); + } else if let Some(array_of_ids) = value.as_array() { + let amount_of_negative_ids = array_of_ids + .iter() + .filter(|id| id.as_i64().is_some_and(|id| id < 0)) + .count(); + increment_mapped_value_by(map, String::from(current_model), amount_of_negative_ids); + } + } else if let serde_json::Value::Object(inner) = value { + count_negative_ids_in_map(map, key, inner); + } + } +} + +/// Increments the value of the key in the given map by the given amount. +fn increment_mapped_value_by< + K: std::cmp::Eq + std::hash::Hash, + V: Default + std::ops::AddAssign, +>( + map: &mut HashMap, + key: K, + amount: V, +) { + *map.entry(key).or_default() += amount; +} + #[derive(Debug, PartialEq)] pub enum DataMutation { Create, @@ -429,19 +525,39 @@ fn capture_mutation_test() { ctx.start_capture().unwrap(); - assert_eq!(ctx.request(String::from(r#" + assert_eq!( + ctx.request( + String::from( + r#" mutation ($input: userInput, $validationSets: [String]) { createuser(input: $input, validationSets: $validationSets) { id } - }"#), String::from(r#"{"id": 1}"#)).unwrap().as_str(), r#"{"data": {"createuser": {"id": "1"}}}"#); - - assert_eq!(ctx.request(String::from(r#" + }"# + ), + String::from(r#"{"id": 1}"#) + ) + .unwrap() + .as_str(), + r#"{"data": {"createuser": {"id": "1"}}}"# + ); + + assert_eq!( + ctx.request( + String::from( + r#" mutation ($input: userInput, $validationSets: [String]) { deleteuser(input: $input, validationSets: $validationSets) { id } - }"#), String::from(r#"{"id": 1}"#)).unwrap().as_str(), r#"{"data": {"deleteuser": {"id": "1"}}}"#); + }"# + ), + String::from(r#"{"id": 1}"#) + ) + .unwrap() + .as_str(), + r#"{"data": {"deleteuser": {"id": "1"}}}"# + ); ctx.apply_capture().unwrap(); } diff --git a/helper/data-api/src/lib.rs b/helper/data-api/src/lib.rs index 934408f..59298f8 100644 --- a/helper/data-api/src/lib.rs +++ b/helper/data-api/src/lib.rs @@ -14,8 +14,8 @@ pub mod data_grpc { pub mod context; use crate::context::DataAPIContext; -use crate::data_grpc::data_api_client::DataApiClient; use crate::data_grpc::DataApiRequest; +use crate::data_grpc::data_api_client::DataApiClient; use crate::data_grpc::{Context as GrpcContext, DataApiResult}; wit_bindgen::generate!({ generate_all }); diff --git a/helper/data-api/tests/data_api_integration_test.rs b/helper/data-api/tests/data_api_integration_test.rs index 4353a74..c5a9b35 100644 --- a/helper/data-api/tests/data_api_integration_test.rs +++ b/helper/data-api/tests/data_api_integration_test.rs @@ -10,11 +10,11 @@ use std::{ }; use tokio::task; use tonic::metadata::MetadataMap; -use tonic::{transport::Server, Request, Response}; +use tonic::{Request, Response, transport::Server}; use tracing::info; use uuid::Uuid; -use wash_runtime::host::http::{DevRouter, HttpServer}; use wash_runtime::host::HostApi; +use wash_runtime::host::http::{DevRouter, HttpServer}; use wash_runtime::types::{Component, LocalResources, Workload, WorkloadStartRequest}; use wash_runtime::wit::WitInterface; use wash_runtime::{ @@ -27,9 +27,9 @@ pub mod data_grpc { use data_grpc::DataApiRequest; use data_grpc::{ + DataApiResult, data_api_result::Status, data_api_server::{DataApi, DataApiServer}, - DataApiResult, }; #[path = "common/mod.rs"] From 51f927f892b112b4e91a2b089981a8d83835973c Mon Sep 17 00:00:00 2001 From: Dewinz Date: Wed, 3 Jun 2026 16:45:49 +0200 Subject: [PATCH 07/20] feat: repurpose counting amount of negative ids to replacing negative ids with their reserved counterpart --- helper/data-api/src/context.rs | 76 ++++++++++++++++------------------ 1 file changed, 36 insertions(+), 40 deletions(-) diff --git a/helper/data-api/src/context.rs b/helper/data-api/src/context.rs index 4b29c91..38a2284 100644 --- a/helper/data-api/src/context.rs +++ b/helper/data-api/src/context.rs @@ -133,9 +133,8 @@ impl GuestDataApi for DataAPIContext { // } // } - for (model_name, all_variables) in upsert_manys { - let _model_and_amount_of_negative_ids = - count_negative_ids_in_maps(&model_name, &all_variables); + for (model_name, mut all_variables) in upsert_manys { + replace_negative_ids_in_all_variables(&vec![], &mut all_variables); let query = format!("mutation {{ upsertMany{model_name}(input: $input) {{ id }} }}"); @@ -271,57 +270,54 @@ impl GuestDataApi for DataAPIContext { } } -/// Loops through a vec of serde_json::Maps and counts the amount of negative ids along with which -/// model name they correspond to. -fn count_negative_ids_in_maps( - current_model: &str, - all_variables: &Vec>, -) -> HashMap { - let mut map: HashMap = HashMap::new(); - - for variables in all_variables { - count_negative_ids_in_map(&mut map, current_model, variables); +/// Loops through a vec of serde_json::Maps and replaces all the negative IDs with their reserved +/// counterparts. +fn replace_negative_ids_in_all_variables( + reserved_ids: &Vec, + all_variables: &mut [serde_json::Map], +) { + for variables in all_variables.iter_mut() { + replace_negative_ids_in_variables(reserved_ids, variables); } - - map } -/// Loops through the entries of a serde_json::Map and counts the amount of negative ids along with -/// which model name they correspond to and puts it into the given HashMap. -fn count_negative_ids_in_map( - map: &mut HashMap, - current_model: &str, - variables: &serde_json::Map, +/// Loops through the entries of a serde_json::Map and replaces all the negative IDs with their +/// reserved counterparts. +fn replace_negative_ids_in_variables( + reserved_ids: &Vec, + variables: &mut serde_json::Map, ) { - for (key, value) in variables { + for (key, value) in variables.iter_mut() { if key == "id" { if let Some(id) = value.as_i64() && id < 0 { - increment_mapped_value_by(map, String::from(current_model), 1); - } else if let Some(array_of_ids) = value.as_array() { - let amount_of_negative_ids = array_of_ids - .iter() - .filter(|id| id.as_i64().is_some_and(|id| id < 0)) - .count(); - increment_mapped_value_by(map, String::from(current_model), amount_of_negative_ids); + *value = get_reserved_id_as_value_for_negative_id(reserved_ids, id); + } else if let serde_json::Value::Array(array_of_ids) = value { + for item in array_of_ids.iter_mut() { + if let Some(id) = item.as_i64() + && id < 0 + { + *item = get_reserved_id_as_value_for_negative_id(reserved_ids, id); + } + } } } else if let serde_json::Value::Object(inner) = value { - count_negative_ids_in_map(map, key, inner); + replace_negative_ids_in_variables(reserved_ids, inner); } } } -/// Increments the value of the key in the given map by the given amount. -fn increment_mapped_value_by< - K: std::cmp::Eq + std::hash::Hash, - V: Default + std::ops::AddAssign, ->( - map: &mut HashMap, - key: K, - amount: V, -) { - *map.entry(key).or_default() += amount; +/// Uses the negative_id to index for its reserved id and put it into a serde_json::Value. +fn get_reserved_id_as_value_for_negative_id( + reserved_ids: &[usize], + negative_id: i64, +) -> serde_json::Value { + // TODO: Do we need guardrails around this to make sure the value is positive and error + // otherwise, or just trust it works? + serde_json::Value::Number(serde_json::Number::from( + reserved_ids[(-1 - negative_id) as usize], + )) } #[derive(Debug, PartialEq)] From c6b64ace7423604dc5791e9ea0df71020c69e753 Mon Sep 17 00:00:00 2001 From: Dewinz Date: Wed, 3 Jun 2026 17:06:18 +0200 Subject: [PATCH 08/20] doc: remove unnecessary TODO and add extra explanation for getting the reserved ID based on the negative ID --- helper/data-api/src/context.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helper/data-api/src/context.rs b/helper/data-api/src/context.rs index 38a2284..ce5d22d 100644 --- a/helper/data-api/src/context.rs +++ b/helper/data-api/src/context.rs @@ -313,9 +313,9 @@ fn get_reserved_id_as_value_for_negative_id( reserved_ids: &[usize], negative_id: i64, ) -> serde_json::Value { - // TODO: Do we need guardrails around this to make sure the value is positive and error - // otherwise, or just trust it works? serde_json::Value::Number(serde_json::Number::from( + // If `negative_id` is -1 which is the first possible negative ID, then `-1 - -1 = 0`. + // Which is the first reserved ID. reserved_ids[(-1 - negative_id) as usize], )) } From 22df8158be042f72c6dee31941188b626fc14738 Mon Sep 17 00:00:00 2001 From: Dewinz Date: Wed, 3 Jun 2026 17:13:26 +0200 Subject: [PATCH 09/20] refactor: move the replacing of negative IDs with reserved IDs to the upsert_manys definition --- helper/data-api/src/context.rs | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/helper/data-api/src/context.rs b/helper/data-api/src/context.rs index ce5d22d..648879e 100644 --- a/helper/data-api/src/context.rs +++ b/helper/data-api/src/context.rs @@ -1,5 +1,4 @@ use std::collections::{HashMap, VecDeque}; -use std::default; use std::sync::{Arc, Mutex}; use crate::exports::betty_blocks::data_api::data_api::{self, GuestDataApi, JsonString}; @@ -115,7 +114,10 @@ impl GuestDataApi for DataAPIContext { // but this is left out of cope for now. let upsert_manys = [create, update, upsert].into_iter().flatten().fold( HashMap::new(), - |mut map, item| { + |mut map, mut item| { + // TODO: Add reserved_ids. + replace_negative_ids_in_variables(&vec![], &mut item.variables); + map.entry(item.model_name) .or_insert(Vec::new()) .push(item.variables); @@ -133,9 +135,7 @@ impl GuestDataApi for DataAPIContext { // } // } - for (model_name, mut all_variables) in upsert_manys { - replace_negative_ids_in_all_variables(&vec![], &mut all_variables); - + for (model_name, all_variables) in upsert_manys { let query = format!("mutation {{ upsertMany{model_name}(input: $input) {{ id }} }}"); @@ -270,17 +270,6 @@ impl GuestDataApi for DataAPIContext { } } -/// Loops through a vec of serde_json::Maps and replaces all the negative IDs with their reserved -/// counterparts. -fn replace_negative_ids_in_all_variables( - reserved_ids: &Vec, - all_variables: &mut [serde_json::Map], -) { - for variables in all_variables.iter_mut() { - replace_negative_ids_in_variables(reserved_ids, variables); - } -} - /// Loops through the entries of a serde_json::Map and replaces all the negative IDs with their /// reserved counterparts. fn replace_negative_ids_in_variables( From 3d4939ab4c4e69cb9a9d5c6c34e422774b87d01d Mon Sep 17 00:00:00 2001 From: Dewinz Date: Thu, 4 Jun 2026 10:15:15 +0200 Subject: [PATCH 10/20] feat: add handling for an array of objects for MassMutateEntry --- helper/data-api/src/context.rs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/helper/data-api/src/context.rs b/helper/data-api/src/context.rs index 648879e..a181825 100644 --- a/helper/data-api/src/context.rs +++ b/helper/data-api/src/context.rs @@ -291,8 +291,22 @@ fn replace_negative_ids_in_variables( } } } - } else if let serde_json::Value::Object(inner) = value { - replace_negative_ids_in_variables(reserved_ids, inner); + } else if let serde_json::Value::Object(object) = value { + replace_negative_ids_in_variables(reserved_ids, object); + } else if let serde_json::Value::Array(array) = value { + if array.is_empty() { + // TODO: Do we return an error? + eprintln!("huh"); + } else { + for item in array.iter_mut() { + if let serde_json::Value::Object(object) = item { + replace_negative_ids_in_variables(reserved_ids, object); + } else { + // TODO: Do we return an error? + eprintln!("huh"); + } + } + } } } } From 69cb71bfe8d73085d1f6a26c0e23cca7f5364288 Mon Sep 17 00:00:00 2001 From: Dewinz Date: Thu, 4 Jun 2026 10:56:13 +0200 Subject: [PATCH 11/20] refactor: split up "replace_negative_ids_in_variables" functionality into multiple functions for readability --- helper/data-api/src/context.rs | 66 ++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/helper/data-api/src/context.rs b/helper/data-api/src/context.rs index a181825..9c4178a 100644 --- a/helper/data-api/src/context.rs +++ b/helper/data-api/src/context.rs @@ -278,35 +278,33 @@ fn replace_negative_ids_in_variables( ) { for (key, value) in variables.iter_mut() { if key == "id" { - if let Some(id) = value.as_i64() - && id < 0 - { - *value = get_reserved_id_as_value_for_negative_id(reserved_ids, id); - } else if let serde_json::Value::Array(array_of_ids) = value { - for item in array_of_ids.iter_mut() { - if let Some(id) = item.as_i64() - && id < 0 - { - *item = get_reserved_id_as_value_for_negative_id(reserved_ids, id); - } - } - } + handle_id_key(reserved_ids, value) } else if let serde_json::Value::Object(object) = value { replace_negative_ids_in_variables(reserved_ids, object); - } else if let serde_json::Value::Array(array) = value { - if array.is_empty() { - // TODO: Do we return an error? - eprintln!("huh"); - } else { - for item in array.iter_mut() { - if let serde_json::Value::Object(object) = item { - replace_negative_ids_in_variables(reserved_ids, object); - } else { - // TODO: Do we return an error? - eprintln!("huh"); - } - } - } + } else if let serde_json::Value::Array(array_of_values) = value { + handle_array_of_values(reserved_ids, array_of_values); + } + } +} + +/// Replaces the negative ID or IDs with their reserved counterpart. +fn handle_id_key(reserved_ids: &[usize], value: &mut serde_json::Value) { + if let Some(id) = value.as_i64() + && id.is_negative() + { + *value = get_reserved_id_as_value_for_negative_id(reserved_ids, id); + } else if let serde_json::Value::Array(array_of_ids) = value { + handle_array_of_ids(reserved_ids, array_of_ids); + } +} + +/// Loops over an Vec of IDs and replaces negative ones with their reserved counterpart. +fn handle_array_of_ids(reserved_ids: &[usize], array_of_ids: &mut [serde_json::Value]) { + for item in array_of_ids.iter_mut() { + if let Some(id) = item.as_i64() + && id.is_negative() + { + *item = get_reserved_id_as_value_for_negative_id(reserved_ids, id); } } } @@ -323,6 +321,20 @@ fn get_reserved_id_as_value_for_negative_id( )) } +/// Looks at the items in an array and if they're an object tries to replace negative IDs in that +/// object, and if they are an array it searches for objects or arrays within that object which +/// might have negative IDs to replace. +fn handle_array_of_values(reserved_ids: &Vec, array_of_values: &mut [serde_json::Value]) { + for item in array_of_values.iter_mut() { + if let serde_json::Value::Object(object) = item { + replace_negative_ids_in_variables(reserved_ids, object); + } else if let serde_json::Value::Array(array) = item { + handle_array_of_values(reserved_ids, array); + } + } +} + + #[derive(Debug, PartialEq)] pub enum DataMutation { Create, From 2c357b1e0457f32ccaf07b8d95d3896c2e42abca Mon Sep 17 00:00:00 2001 From: Dewinz Date: Thu, 4 Jun 2026 10:57:34 +0200 Subject: [PATCH 12/20] chore: run cargo fmt --- helper/data-api/src/context.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/helper/data-api/src/context.rs b/helper/data-api/src/context.rs index 9c4178a..7cb00fa 100644 --- a/helper/data-api/src/context.rs +++ b/helper/data-api/src/context.rs @@ -334,7 +334,6 @@ fn handle_array_of_values(reserved_ids: &Vec, array_of_values: &mut [serd } } - #[derive(Debug, PartialEq)] pub enum DataMutation { Create, From a4ed5a1d8726d23a2d3f3383adb8ab133be85076 Mon Sep 17 00:00:00 2001 From: Dewinz Date: Thu, 4 Jun 2026 10:59:54 +0200 Subject: [PATCH 13/20] doc: rewrite negative ID to reserved ID handling to be clearer --- helper/data-api/src/context.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helper/data-api/src/context.rs b/helper/data-api/src/context.rs index 7cb00fa..bf948dd 100644 --- a/helper/data-api/src/context.rs +++ b/helper/data-api/src/context.rs @@ -316,7 +316,7 @@ fn get_reserved_id_as_value_for_negative_id( ) -> serde_json::Value { serde_json::Value::Number(serde_json::Number::from( // If `negative_id` is -1 which is the first possible negative ID, then `-1 - -1 = 0`. - // Which is the first reserved ID. + // Which is the first possible item in the reserved ID list. reserved_ids[(-1 - negative_id) as usize], )) } From 250b531f605688931b3c382e609d51ea7a031f3b Mon Sep 17 00:00:00 2001 From: Dewinz Date: Thu, 4 Jun 2026 11:20:11 +0200 Subject: [PATCH 14/20] refactor: use TryInto for conversion of negative ID to reserved ID for WASM 32 bit limit --- helper/data-api/src/context.rs | 49 +++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/helper/data-api/src/context.rs b/helper/data-api/src/context.rs index bf948dd..d2f5447 100644 --- a/helper/data-api/src/context.rs +++ b/helper/data-api/src/context.rs @@ -116,7 +116,8 @@ impl GuestDataApi for DataAPIContext { HashMap::new(), |mut map, mut item| { // TODO: Add reserved_ids. - replace_negative_ids_in_variables(&vec![], &mut item.variables); + // TODO: Remove unwrap. + replace_negative_ids_in_variables(&vec![], &mut item.variables).unwrap(); map.entry(item.model_name) .or_insert(Vec::new()) @@ -275,63 +276,73 @@ impl GuestDataApi for DataAPIContext { fn replace_negative_ids_in_variables( reserved_ids: &Vec, variables: &mut serde_json::Map, -) { +) -> Result<(), String> { for (key, value) in variables.iter_mut() { if key == "id" { - handle_id_key(reserved_ids, value) + handle_id_key(reserved_ids, value)?; } else if let serde_json::Value::Object(object) = value { - replace_negative_ids_in_variables(reserved_ids, object); + replace_negative_ids_in_variables(reserved_ids, object)?; } else if let serde_json::Value::Array(array_of_values) = value { - handle_array_of_values(reserved_ids, array_of_values); + handle_array_of_values(reserved_ids, array_of_values)?; } } + + Ok(()) } /// Replaces the negative ID or IDs with their reserved counterpart. -fn handle_id_key(reserved_ids: &[usize], value: &mut serde_json::Value) { +fn handle_id_key(reserved_ids: &[usize], value: &mut serde_json::Value) -> Result<(), String> { if let Some(id) = value.as_i64() && id.is_negative() { - *value = get_reserved_id_as_value_for_negative_id(reserved_ids, id); + *value = get_reserved_id_as_value_for_negative_id(reserved_ids, id)?; } else if let serde_json::Value::Array(array_of_ids) = value { - handle_array_of_ids(reserved_ids, array_of_ids); + handle_array_of_ids(reserved_ids, array_of_ids)?; } + + Ok(()) } /// Loops over an Vec of IDs and replaces negative ones with their reserved counterpart. -fn handle_array_of_ids(reserved_ids: &[usize], array_of_ids: &mut [serde_json::Value]) { +fn handle_array_of_ids(reserved_ids: &[usize], array_of_ids: &mut [serde_json::Value]) -> Result<(), String> { for item in array_of_ids.iter_mut() { if let Some(id) = item.as_i64() && id.is_negative() { - *item = get_reserved_id_as_value_for_negative_id(reserved_ids, id); + *item = get_reserved_id_as_value_for_negative_id(reserved_ids, id)?; } } + + Ok(()) } /// Uses the negative_id to index for its reserved id and put it into a serde_json::Value. fn get_reserved_id_as_value_for_negative_id( reserved_ids: &[usize], negative_id: i64, -) -> serde_json::Value { - serde_json::Value::Number(serde_json::Number::from( - // If `negative_id` is -1 which is the first possible negative ID, then `-1 - -1 = 0`. - // Which is the first possible item in the reserved ID list. - reserved_ids[(-1 - negative_id) as usize], - )) +) -> Result { + // If `negative_id` is -1 which is the first possible negative ID, then `-1 - -1 = 0`. + // Which is the first possible item in the reserved ID list. + let index: usize = (-1 - negative_id).try_into().map_err(|error| format!("could not convert number to usize: {error}"))?; + + Ok(serde_json::Value::Number(serde_json::Number::from( + reserved_ids[index], + ))) } /// Looks at the items in an array and if they're an object tries to replace negative IDs in that /// object, and if they are an array it searches for objects or arrays within that object which /// might have negative IDs to replace. -fn handle_array_of_values(reserved_ids: &Vec, array_of_values: &mut [serde_json::Value]) { +fn handle_array_of_values(reserved_ids: &Vec, array_of_values: &mut [serde_json::Value]) -> Result<(), String> { for item in array_of_values.iter_mut() { if let serde_json::Value::Object(object) = item { - replace_negative_ids_in_variables(reserved_ids, object); + replace_negative_ids_in_variables(reserved_ids, object)?; } else if let serde_json::Value::Array(array) = item { - handle_array_of_values(reserved_ids, array); + handle_array_of_values(reserved_ids, array)?; } } + + Ok(()) } #[derive(Debug, PartialEq)] From 195d59855fab489a75569fa783d4748acee4e317 Mon Sep 17 00:00:00 2001 From: Robin Walthuis Date: Fri, 5 Jun 2026 09:24:30 +0200 Subject: [PATCH 15/20] feat: add id reserving logic --- helper/data-api/Cargo.lock | 1 + helper/data-api/Cargo.toml | 1 + helper/data-api/src/context.rs | 247 ++++++++++++++++++++++----------- 3 files changed, 171 insertions(+), 78 deletions(-) diff --git a/helper/data-api/Cargo.lock b/helper/data-api/Cargo.lock index 7b649a5..688f4c4 100644 --- a/helper/data-api/Cargo.lock +++ b/helper/data-api/Cargo.lock @@ -719,6 +719,7 @@ dependencies = [ "prost 0.13.5", "reqwest", "semver", + "serde", "serde_json", "serial_test", "tokio", diff --git a/helper/data-api/Cargo.toml b/helper/data-api/Cargo.toml index c8dee32..387cd75 100644 --- a/helper/data-api/Cargo.toml +++ b/helper/data-api/Cargo.toml @@ -23,6 +23,7 @@ uuid = { version = "1.8.0", features = ["v4"] } tracing = "0.1" graphql-parser = "0.4.1" serde_json = "1.0.145" +serde = { version = "1.0.228", features = ["derive"] } [dev-dependencies] wash-runtime = { git = "https://github.com/bettyblocks/wash", features = [ diff --git a/helper/data-api/src/context.rs b/helper/data-api/src/context.rs index d2f5447..7d0b698 100644 --- a/helper/data-api/src/context.rs +++ b/helper/data-api/src/context.rs @@ -1,9 +1,13 @@ -use std::collections::{HashMap, VecDeque}; +use std::collections::{BTreeMap, HashMap, VecDeque}; use std::sync::{Arc, Mutex}; use crate::exports::betty_blocks::data_api::data_api::{self, GuestDataApi, JsonString}; use crate::{Config, inner_request}; +type ModelName = String; +type InternalId = isize; +type RealId = i32; + pub struct DataAPIContext { application_id: String, @@ -12,19 +16,25 @@ pub struct DataAPIContext { action_id: String, /// jwt of the customer (so not the jaws jwt used for authenticating the server to server communication) jwt: Option, - capture_stack: Arc>>, + capture_data: Arc>, +} + +#[derive(Debug, Default)] +pub struct CaptureData { + model_names_of_local_ids: Vec, + reserve_id_count_per_model: HashMap, + capture_stack: Vec, } #[derive(Default, Debug)] pub struct MassMutateEntries { create: VecDeque, update: VecDeque, - upsert: VecDeque, delete: VecDeque, } impl MassMutateEntries { - fn as_pending_mutations(&self) -> [Vec; 4] { + fn as_pending_mutations(&self) -> [Vec; 3] { [ self.create .iter() @@ -34,10 +44,6 @@ impl MassMutateEntries { .iter() .map(|x| x.as_pending_mutation(DataMutation::Update)) .collect(), - self.upsert - .iter() - .map(|x| x.as_pending_mutation(DataMutation::Upsert)) - .collect(), self.delete .iter() .map(|x| x.as_pending_mutation(DataMutation::Delete)) @@ -62,6 +68,22 @@ impl MassMutateEntry { } } +#[derive(serde::Serialize, serde::Deserialize)] +pub struct ReserveIdMutationResult { + data: ReserveIdResult, +} + +#[derive(serde::Serialize, serde::Deserialize)] +pub struct ReserveIdResult { + #[serde(rename = "reserveRecords")] + reserved_ids: ReservedIds +} + +#[derive(serde::Serialize, serde::Deserialize)] +pub struct ReservedIds { + ids: VecDeque +} + fn format_mutation(model_name: &str, operation: &DataMutation) -> String { format!( "mutation {{ {}{model_name}(input: $input) {{ id }} }}", @@ -77,24 +99,48 @@ fn generate_delayed_id(id: &str, mutation_name: &str) -> JsonString { format!(r#"{{"data": {{"{mutation_name}": {{"id": "{id}"}}}}}}"#) } +fn reserve_ids(data_api_context: &impl GuestDataApi, (model_name, id_count): (ModelName, usize)) -> Result<(ModelName, VecDeque), String> { + let query = + r#"mutation ($model: String!, $amount: Int!) { + reserveRecords(model: $model, amount: $amount) { + ids + } + }"#; + + let variables = format!(r#"{{"model": "{model_name}", "amount": {id_count}}}"#); + + let res = data_api_context.request_raw(query.to_string(), variables)?; + + let ReserveIdMutationResult{data: ReserveIdResult { reserved_ids: ReservedIds { ids } }} = serde_json::from_str(&res).map_err(|_| String::from("could not parse data api result for reserving ids"))?; + + Ok((model_name, ids)) +} + impl GuestDataApi for DataAPIContext { fn new(application_id: String, action_id: String, jwt: Option) -> Self { DataAPIContext { application_id, action_id, jwt, - capture_stack: Default::default(), + capture_data: Default::default(), } } fn apply_capture(&self) -> Result { + let mut capture_data = self + .capture_data + .lock() + .map_err(|_| String::from("capture stack lock poisoned"))?; if let Some(MassMutateEntries { create, update, - upsert, delete, - }) = self.capture_stack.lock().expect("lock is poisoned").pop() + }) = capture_data.capture_stack.pop() { + let mut reserved_id_map = capture_data.reserve_id_count_per_model.drain().map(|entry| reserve_ids(self, entry)).collect::>, String>>()?; + + let reserved_ids = capture_data.model_names_of_local_ids.iter().map(|model_name| Ok(reserved_id_map.get_mut(model_name).ok_or_else(|| format!("ids for model {model_name} were not properly reserved"))?.pop_front().ok_or_else(|| format!("not enough ids were reserved for model {model_name}"))?)).collect::, String>>()?; + /* TODO: resolve ids Locally generated ids need to be distinguishable from normal ids after being sent back to the caller and then sent here as for example a related row. This will probably be done with negative numbered ids? Unless we have a better way of distinguising them. @@ -112,19 +158,20 @@ impl GuestDataApi for DataAPIContext { // If we want to be able to specify a unique by for the upsert we would want to handle them separately, // but this is left out of cope for now. - let upsert_manys = [create, update, upsert].into_iter().flatten().fold( - HashMap::new(), - |mut map, mut item| { - // TODO: Add reserved_ids. - // TODO: Remove unwrap. - replace_negative_ids_in_variables(&vec![], &mut item.variables).unwrap(); - - map.entry(item.model_name) - .or_insert(Vec::new()) - .push(item.variables); - map - }, - ); + let upsert_manys = + [create, update] + .into_iter() + .flatten() + .fold(HashMap::new(), |mut map, mut item| { + // TODO: Add reserved_ids. + // TODO: Remove unwrap. + replace_negative_ids_in_variables(&vec![], &mut item.variables).unwrap(); + + map.entry(item.model_name) + .or_insert(Vec::new()) + .push(item.variables); + map + }); // TODO: Do we want to delete entries before they get upserted where possible? // Would mean less unnecessary mutations in the data api, but likely worse performance if we need to check for that here. @@ -136,13 +183,13 @@ impl GuestDataApi for DataAPIContext { // } // } - for (model_name, all_variables) in upsert_manys { + for (model_name, variables) in upsert_manys { let query = format!("mutation {{ upsertMany{model_name}(input: $input) {{ id }} }}"); let variables = format!( "{{\"input\": {}}}", - serde_json::to_string(&all_variables) + serde_json::to_string(&variables) .map_err(|_| String::from("could not format input variables"))? ); @@ -177,7 +224,13 @@ impl GuestDataApi for DataAPIContext { } fn discard_capture(&self) -> Result { - if let Some(_) = self.capture_stack.lock().expect("lock is poisoned").pop() { + if let Some(_) = self + .capture_data + .lock() + .expect("lock is poisoned") + .capture_stack + .pop() + { return Ok(format!("deleted most recent capture stack entry")); } @@ -185,15 +238,22 @@ impl GuestDataApi for DataAPIContext { } fn start_capture(&self) -> Result<(), String> { - self.capture_stack + self.capture_data .lock() .map_err(|_| String::from("capture stack lock poisoned"))? - .push(MassMutateEntries::default()); + .capture_stack + .push(Default::default()); Ok(()) } fn pending_capture(&self) -> Result>, String> { - if let Some(entries) = self.capture_stack.lock().expect("lock is poisoned").last() { + if let Some(entries) = self + .capture_data + .lock() + .expect("lock is poisoned") + .capture_stack + .last() + { return Ok(entries.as_pending_mutations().to_vec()); } @@ -201,45 +261,86 @@ impl GuestDataApi for DataAPIContext { } fn request(&self, query: String, variables: JsonString) -> Result { - if let Ok(mut capture_stack) = self.capture_stack.lock() { - if let Some(entries) = capture_stack.last_mut() { - let query_remainder = match query.split_once("mutation") { - Some((_, remainder)) => remainder, - None => return self.request_raw(query, variables), - }; - - let query_remainder = query_remainder - .split_once('{') - .ok_or_else(|| String::from("query is improperly formatted"))? - .1 - .trim(); - - if query_remainder[6..10] == *"Many" { - return self.request_raw(query, variables); - } + if let Ok(mut capture_data) = self.capture_data.lock() + && capture_data.capture_stack.len() != 0 + { + let query_remainder = match query.split_once("mutation") { + Some((_, remainder)) => remainder, + None => return self.request_raw(query, variables), + }; + + let query_remainder = query_remainder + .split_once('{') + .ok_or_else(|| String::from("query is improperly formatted"))? + .1 + .trim(); + + if query_remainder[6..10] == *"Many" { + return self.request_raw(query, variables); + } - let mutation_name = query_remainder - .split_once(|c: char| c.is_whitespace() || c == '(') - .ok_or_else(|| String::from("query is improperly formatted"))? - .0; - - match &query_remainder[0..6] { - "create" => &mut entries.create, - "update" => &mut entries.update, - "upsert" => &mut entries.upsert, - "delete" => &mut entries.delete, - _ => return self.request_raw(query, variables), + let mutation_name = query_remainder + .split_once(|c: char| c.is_whitespace() || c == '(') + .ok_or_else(|| String::from("query is improperly formatted"))? + .0; + + let model_name = mutation_name[6..].to_string(); + + let mass_mutate_entry = MassMutateEntry { + model_name: model_name.clone(), + variables: serde_json::from_str(&variables) + .map_err(|_| String::from("could not parse variables"))?, + }; + + // It is safe to unwrap the capture stack last_mut here because we checked the length to be non-zero before. This looks a bit wack but is necessary to not mutably borrow capture_data multiple times. + let id: isize = match &query_remainder[0..6] { + "create" => { + // TODO: you could specify an id with a create, that isn't handled yet. + capture_data + .model_names_of_local_ids + .push(model_name.clone()); + capture_data + .reserve_id_count_per_model + .entry(model_name) + .and_modify(|x| *x += 1); + capture_data + .capture_stack + .last_mut() + .unwrap() + .create + .push_back(mass_mutate_entry); + -capture_data + .capture_stack + .len() + .try_into() + .map_err(|_| String::from("ran out of internal ids"))? } - .push_back(MassMutateEntry { - model_name: mutation_name[6..].to_string(), - variables: serde_json::from_str(&variables) - .map_err(|_| String::from("could not parse variables"))?, - }); + "update" => { + capture_data + .capture_stack + .last_mut() + .unwrap() + .update + .push_back(mass_mutate_entry); + // TODO: find the id and return it. + 1isize + } + "delete" => { + capture_data + .capture_stack + .last_mut() + .unwrap() + .delete + .push_back(mass_mutate_entry); + // TODO: find the id and return it. + 1isize + } + _ => return self.request_raw(query, variables), + }; - // TODO: Generate a local id that will be resolved to a real id later. - // Read more about this logic in [[Self::apply_capture]] - return Ok(generate_delayed_id("1", mutation_name)); - } + // TODO: Generate a local id that will be resolved to a real id later. + // Read more about this logic in [[Self::apply_capture]] + return Ok(generate_delayed_id(&id.to_string(), mutation_name)); } self.request_raw(query, variables) @@ -349,7 +450,6 @@ fn handle_array_of_values(reserved_ids: &Vec, array_of_values: &mut [serd pub enum DataMutation { Create, Update, - Upsert, Delete, } @@ -360,7 +460,6 @@ impl DataMutation { match self { Create => "create", Update => "update", - Upsert => "upsert", Delete => "delete", } } @@ -390,14 +489,6 @@ impl Instruction { } } - pub fn upsert(name: String) -> Self { - Instruction { - name, - operator: DataMutation::Upsert, - data: None, - } - } - pub fn delete(name: String) -> Self { Instruction { name, @@ -556,7 +647,7 @@ fn capture_mutation_test() { } }"# ), - String::from(r#"{"id": 1}"#) + String::from(r#"{"input": {"id": 1}}"#) ) .unwrap() .as_str(), @@ -573,7 +664,7 @@ fn capture_mutation_test() { } }"# ), - String::from(r#"{"id": 1}"#) + String::from(r#"{"input": {"id": 1}}"#) ) .unwrap() .as_str(), From eddb190384d74d40d7300f6888f7ee8dfc1b5aff Mon Sep 17 00:00:00 2001 From: Robin Walthuis Date: Fri, 5 Jun 2026 11:45:12 +0200 Subject: [PATCH 16/20] fix: resolve some todos --- helper/data-api/src/context.rs | 362 ++++++++++++++++++--------------- 1 file changed, 203 insertions(+), 159 deletions(-) diff --git a/helper/data-api/src/context.rs b/helper/data-api/src/context.rs index 7d0b698..5688cd2 100644 --- a/helper/data-api/src/context.rs +++ b/helper/data-api/src/context.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeMap, HashMap, VecDeque}; +use std::collections::{HashMap, VecDeque}; use std::sync::{Arc, Mutex}; use crate::exports::betty_blocks::data_api::data_api::{self, GuestDataApi, JsonString}; @@ -28,6 +28,7 @@ pub struct CaptureData { #[derive(Default, Debug)] pub struct MassMutateEntries { + // TODO: We are probably able to turn these into Vecs because we iterate through them front-to-back anyway create: VecDeque, update: VecDeque, delete: VecDeque, @@ -76,12 +77,24 @@ pub struct ReserveIdMutationResult { #[derive(serde::Serialize, serde::Deserialize)] pub struct ReserveIdResult { #[serde(rename = "reserveRecords")] - reserved_ids: ReservedIds + reserved_ids: ReservedIds, } #[derive(serde::Serialize, serde::Deserialize)] pub struct ReservedIds { - ids: VecDeque + ids: VecDeque, +} + +#[derive(serde::Serialize, serde::Deserialize)] +pub struct MutationInput { + input: MutationInputVariable, +} + +#[derive(serde::Serialize, serde::Deserialize)] +pub struct MutationInputVariable { + id: InternalId, + #[serde(flatten)] + other_inputs: HashMap, } fn format_mutation(model_name: &str, operation: &DataMutation) -> String { @@ -95,25 +108,32 @@ fn format_mutation_name(model_name: &str, operation: &DataMutation) -> String { format!("{}{model_name}", operation.as_static_str()) } -fn generate_delayed_id(id: &str, mutation_name: &str) -> JsonString { +fn generate_delayed_id_response(id: &str, mutation_name: &str) -> JsonString { format!(r#"{{"data": {{"{mutation_name}": {{"id": "{id}"}}}}}}"#) } -fn reserve_ids(data_api_context: &impl GuestDataApi, (model_name, id_count): (ModelName, usize)) -> Result<(ModelName, VecDeque), String> { - let query = - r#"mutation ($model: String!, $amount: Int!) { +fn reserve_ids( + data_api_context: &impl GuestDataApi, + (model_name, id_count): (ModelName, usize), +) -> Result<(ModelName, VecDeque), String> { + let query = r#"mutation ($model: String!, $amount: Int!) { reserveRecords(model: $model, amount: $amount) { ids } }"#; - let variables = format!(r#"{{"model": "{model_name}", "amount": {id_count}}}"#); + let variables = format!(r#"{{"model": "{model_name}", "amount": {id_count}}}"#); - let res = data_api_context.request_raw(query.to_string(), variables)?; + let res = data_api_context.request_raw(query.to_string(), variables)?; - let ReserveIdMutationResult{data: ReserveIdResult { reserved_ids: ReservedIds { ids } }} = serde_json::from_str(&res).map_err(|_| String::from("could not parse data api result for reserving ids"))?; + let ReserveIdMutationResult { + data: ReserveIdResult { + reserved_ids: ReservedIds { ids }, + }, + } = serde_json::from_str(&res) + .map_err(|_| String::from("could not parse data api result for reserving ids"))?; - Ok((model_name, ids)) + Ok((model_name, ids)) } impl GuestDataApi for DataAPIContext { @@ -137,11 +157,29 @@ impl GuestDataApi for DataAPIContext { delete, }) = capture_data.capture_stack.pop() { - let mut reserved_id_map = capture_data.reserve_id_count_per_model.drain().map(|entry| reserve_ids(self, entry)).collect::>, String>>()?; - - let reserved_ids = capture_data.model_names_of_local_ids.iter().map(|model_name| Ok(reserved_id_map.get_mut(model_name).ok_or_else(|| format!("ids for model {model_name} were not properly reserved"))?.pop_front().ok_or_else(|| format!("not enough ids were reserved for model {model_name}"))?)).collect::, String>>()?; - - /* TODO: resolve ids + let mut reserved_id_map = capture_data + .reserve_id_count_per_model + .drain() + .map(|entry| reserve_ids(self, entry)) + .collect::>, String>>()?; + + let reserved_ids = capture_data + .model_names_of_local_ids + .iter() + .map(|model_name| { + Ok(reserved_id_map + .get_mut(model_name) + .ok_or_else(|| { + format!("ids for model {model_name} were not properly reserved") + })? + .pop_front() + .ok_or_else(|| { + format!("not enough ids were reserved for model {model_name}") + })? as RealId) + }) + .collect::, String>>()?; + + /* Locally generated ids need to be distinguishable from normal ids after being sent back to the caller and then sent here as for example a related row. This will probably be done with negative numbered ids? Unless we have a better way of distinguising them. Preferrably the caller wouldn't be able to distinguish them. @@ -156,32 +194,23 @@ impl GuestDataApi for DataAPIContext { These can be clustered by model, so we can do upsertManyUser but not upsertManyUserAndRole. */ + let mut upsert_manys = HashMap::new(); + // If we want to be able to specify a unique by for the upsert we would want to handle them separately, - // but this is left out of cope for now. - let upsert_manys = - [create, update] - .into_iter() - .flatten() - .fold(HashMap::new(), |mut map, mut item| { - // TODO: Add reserved_ids. - // TODO: Remove unwrap. - replace_negative_ids_in_variables(&vec![], &mut item.variables).unwrap(); - - map.entry(item.model_name) - .or_insert(Vec::new()) - .push(item.variables); - map - }); - - // TODO: Do we want to delete entries before they get upserted where possible? - // Would mean less unnecessary mutations in the data api, but likely worse performance if we need to check for that here. - // It would look something like this. - // for entry in delete { - // if let Some(id) = entry.variables.get("id") && let Some(model_entry) = x.get_mut(&entry.model_name) { - // model_entry.retain(|v| !v.get("id").is_some_and(|y| y == id)) - // // Still need to delete if this doesn't match anything - // } - // } + // but this is left out of scope for now. + + for MassMutateEntry { + model_name, + mut variables, + } in [create, update].into_iter().flatten() + { + replace_negative_ids_in_variables(&reserved_ids, &mut variables).unwrap(); + + upsert_manys + .entry(model_name) + .or_insert(Vec::new()) + .push(variables); + } for (model_name, variables) in upsert_manys { let query = @@ -193,8 +222,8 @@ impl GuestDataApi for DataAPIContext { .map_err(|_| String::from("could not format input variables"))? ); - // TODO: set up some kind of mocking so this doesn't break when testing. Same goes for the delete manys. - // self.request_raw(query, variables)?; + // TODO: set up some kind of mocking so this doesn't break when testing. Same goes for the delete manys and reserve ids. + self.request_raw(query, variables)?; } let delete_manys = delete @@ -216,7 +245,7 @@ impl GuestDataApi for DataAPIContext { .map_err(|_| String::from("could not format input variables"))? ); - // self.request_raw(query, variables)?; + self.request_raw(query, variables)?; } } @@ -224,14 +253,15 @@ impl GuestDataApi for DataAPIContext { } fn discard_capture(&self) -> Result { - if let Some(_) = self + if self .capture_data .lock() .expect("lock is poisoned") .capture_stack .pop() + .is_some() { - return Ok(format!("deleted most recent capture stack entry")); + return Ok(String::from("deleted most recent capture stack entry")); } Ok(String::from("nothing to do")) @@ -262,7 +292,7 @@ impl GuestDataApi for DataAPIContext { fn request(&self, query: String, variables: JsonString) -> Result { if let Ok(mut capture_data) = self.capture_data.lock() - && capture_data.capture_stack.len() != 0 + && !capture_data.capture_stack.is_empty() { let query_remainder = match query.split_once("mutation") { Some((_, remainder)) => remainder, @@ -294,53 +324,74 @@ impl GuestDataApi for DataAPIContext { // It is safe to unwrap the capture stack last_mut here because we checked the length to be non-zero before. This looks a bit wack but is necessary to not mutably borrow capture_data multiple times. let id: isize = match &query_remainder[0..6] { - "create" => { - // TODO: you could specify an id with a create, that isn't handled yet. - capture_data - .model_names_of_local_ids - .push(model_name.clone()); - capture_data - .reserve_id_count_per_model - .entry(model_name) - .and_modify(|x| *x += 1); - capture_data - .capture_stack - .last_mut() - .unwrap() - .create - .push_back(mass_mutate_entry); - -capture_data - .capture_stack - .len() - .try_into() - .map_err(|_| String::from("ran out of internal ids"))? - } + "create" => match serde_json::from_str(&variables) { + Ok(MutationInput { + input: MutationInputVariable { id, .. }, + }) => { + capture_data + .capture_stack + .last_mut() + .unwrap() + .create + .push_back(mass_mutate_entry); + + id + } + Err(_) => { + capture_data + .model_names_of_local_ids + .push(model_name.clone()); + *capture_data + .reserve_id_count_per_model + .entry(model_name) + .or_default() += 1; + capture_data + .capture_stack + .last_mut() + .unwrap() + .create + .push_back(mass_mutate_entry); + -capture_data + .capture_stack + .len() + .try_into() + .map_err(|_| String::from("ran out of internal ids"))? + } + }, "update" => { + let MutationInput { + input: MutationInputVariable { id, .. }, + } = serde_json::from_str(&variables) + .map_err(|_| String::from("could not find id input for update query"))?; + capture_data .capture_stack .last_mut() .unwrap() .update .push_back(mass_mutate_entry); - // TODO: find the id and return it. - 1isize + + id } "delete" => { + let MutationInput { + input: MutationInputVariable { id, .. }, + } = serde_json::from_str(&variables) + .map_err(|_| String::from("could not find id input for delete query"))?; + capture_data .capture_stack .last_mut() .unwrap() .delete .push_back(mass_mutate_entry); - // TODO: find the id and return it. - 1isize + + id } _ => return self.request_raw(query, variables), }; - // TODO: Generate a local id that will be resolved to a real id later. - // Read more about this logic in [[Self::apply_capture]] - return Ok(generate_delayed_id(&id.to_string(), mutation_name)); + return Ok(generate_delayed_id_response(&id.to_string(), mutation_name)); } self.request_raw(query, variables) @@ -375,7 +426,7 @@ impl GuestDataApi for DataAPIContext { /// Loops through the entries of a serde_json::Map and replaces all the negative IDs with their /// reserved counterparts. fn replace_negative_ids_in_variables( - reserved_ids: &Vec, + reserved_ids: &[RealId], variables: &mut serde_json::Map, ) -> Result<(), String> { for (key, value) in variables.iter_mut() { @@ -392,7 +443,7 @@ fn replace_negative_ids_in_variables( } /// Replaces the negative ID or IDs with their reserved counterpart. -fn handle_id_key(reserved_ids: &[usize], value: &mut serde_json::Value) -> Result<(), String> { +fn handle_id_key(reserved_ids: &[RealId], value: &mut serde_json::Value) -> Result<(), String> { if let Some(id) = value.as_i64() && id.is_negative() { @@ -405,7 +456,10 @@ fn handle_id_key(reserved_ids: &[usize], value: &mut serde_json::Value) -> Resul } /// Loops over an Vec of IDs and replaces negative ones with their reserved counterpart. -fn handle_array_of_ids(reserved_ids: &[usize], array_of_ids: &mut [serde_json::Value]) -> Result<(), String> { +fn handle_array_of_ids( + reserved_ids: &[RealId], + array_of_ids: &mut [serde_json::Value], +) -> Result<(), String> { for item in array_of_ids.iter_mut() { if let Some(id) = item.as_i64() && id.is_negative() @@ -419,12 +473,14 @@ fn handle_array_of_ids(reserved_ids: &[usize], array_of_ids: &mut [serde_json::V /// Uses the negative_id to index for its reserved id and put it into a serde_json::Value. fn get_reserved_id_as_value_for_negative_id( - reserved_ids: &[usize], + reserved_ids: &[RealId], negative_id: i64, ) -> Result { // If `negative_id` is -1 which is the first possible negative ID, then `-1 - -1 = 0`. // Which is the first possible item in the reserved ID list. - let index: usize = (-1 - negative_id).try_into().map_err(|error| format!("could not convert number to usize: {error}"))?; + let index: usize = (-1 - negative_id) + .try_into() + .map_err(|error| format!("could not convert number to usize: {error}"))?; Ok(serde_json::Value::Number(serde_json::Number::from( reserved_ids[index], @@ -434,7 +490,10 @@ fn get_reserved_id_as_value_for_negative_id( /// Looks at the items in an array and if they're an object tries to replace negative IDs in that /// object, and if they are an array it searches for objects or arrays within that object which /// might have negative IDs to replace. -fn handle_array_of_values(reserved_ids: &Vec, array_of_values: &mut [serde_json::Value]) -> Result<(), String> { +fn handle_array_of_values( + reserved_ids: &[RealId], + array_of_values: &mut [serde_json::Value], +) -> Result<(), String> { for item in array_of_values.iter_mut() { if let serde_json::Value::Object(object) = item { replace_negative_ids_in_variables(reserved_ids, object)?; @@ -508,71 +567,62 @@ pub fn parse_graphql_to_intruction(graphql: &str) -> Result, for def in document.definitions { match def { - graphql_parser::query::Definition::Operation(operation) => match operation { - graphql_parser::query::OperationDefinition::Mutation(mutation) => { - if mutation.selection_set.items.len() > 1 { - return Ok(None); - } + graphql_parser::query::Definition::Operation( + graphql_parser::query::OperationDefinition::Mutation(mutation), + ) => { + if mutation.selection_set.items.len() > 1 { + return Ok(None); + } - for item in mutation.selection_set.items { - match item { - graphql_parser::query::Selection::Field(field) - if field.name.starts_with("create") - && !field.name.starts_with("createMany") => - { - let name = field.name.split_at("create".len()).1.to_string(); - let mut new_instruction = Instruction::create(name); - - extract_values_from_gql_argument( - &mut new_instruction, - field.arguments, - ); - - instruction = Some(new_instruction); - } - graphql_parser::query::Selection::Field(field) - if field.name.starts_with("update") - && !field.name.starts_with("updateMany") => - { - let name = field.name.split_at("update".len()).1.to_string(); - let mut new_instruction = Instruction::update(name); - - extract_values_from_gql_argument( - &mut new_instruction, - field.arguments, - ); - - instruction = Some(new_instruction); - } - graphql_parser::query::Selection::Field(field) - if field.name.starts_with("delete") - && !field.name.starts_with("deleteMany") => - { - let name = field.name.split_at("delete".len()).1.to_string(); - let mut new_instruction = Instruction::delete(name); - - extract_values_from_gql_argument( - &mut new_instruction, - field.arguments, - ); - - instruction = Some(new_instruction); - } - _ => {} + for item in mutation.selection_set.items { + match item { + graphql_parser::query::Selection::Field(field) + if field.name.starts_with("create") + && !field.name.starts_with("createMany") => + { + let name = field.name.split_at("create".len()).1.to_string(); + let mut new_instruction = Instruction::create(name); + + extract_values_from_gql_argument(&mut new_instruction, field.arguments); + + instruction = Some(new_instruction); } + graphql_parser::query::Selection::Field(field) + if field.name.starts_with("update") + && !field.name.starts_with("updateMany") => + { + let name = field.name.split_at("update".len()).1.to_string(); + let mut new_instruction = Instruction::update(name); + + extract_values_from_gql_argument(&mut new_instruction, field.arguments); + + instruction = Some(new_instruction); + } + graphql_parser::query::Selection::Field(field) + if field.name.starts_with("delete") + && !field.name.starts_with("deleteMany") => + { + let name = field.name.split_at("delete".len()).1.to_string(); + let mut new_instruction = Instruction::delete(name); + + extract_values_from_gql_argument(&mut new_instruction, field.arguments); + + instruction = Some(new_instruction); + } + _ => {} } } - graphql_parser::query::OperationDefinition::SelectionSet(set) => { - for item in set.items { - match item { - graphql_parser::query::Selection::Field(field) - if field.name == "id" => {} - _ => {} - } + } + graphql_parser::query::Definition::Operation( + graphql_parser::query::OperationDefinition::SelectionSet(set), + ) => { + for item in set.items { + match item { + graphql_parser::query::Selection::Field(field) if field.name == "id" => {} + _ => {} } } - _ => {} - }, + } _ => {} } } @@ -597,27 +647,21 @@ fn extract_values_from_gql_argument( fn graphql_to_json(val: graphql_parser::query::Value<'_, String>) -> Option { match val { - graphql_parser::query::Value::Variable(_) => return None, - graphql_parser::query::Value::Boolean(b) => return Some(serde_json::Value::Bool(b)), - graphql_parser::query::Value::String(s) => { - return Some(serde_json::Value::String(s)); - } + graphql_parser::query::Value::Variable(_) => None, + graphql_parser::query::Value::Boolean(b) => Some(serde_json::Value::Bool(b)), + graphql_parser::query::Value::String(s) => Some(serde_json::Value::String(s)), graphql_parser::query::Value::Int(i) => { let num = i.as_i64()?; - return Some(serde_json::Value::Number(num.into())); + Some(serde_json::Value::Number(num.into())) } graphql_parser::query::Value::Float(f) => { - return Some(serde_json::Value::Number(Number::from_f64(f)?)); - } - graphql_parser::query::Value::Null => { - return Some(serde_json::Value::Null); - } - graphql_parser::query::Value::Enum(s) => { - return Some(serde_json::Value::String(s)); + Some(serde_json::Value::Number(Number::from_f64(f)?)) } + graphql_parser::query::Value::Null => Some(serde_json::Value::Null), + graphql_parser::query::Value::Enum(s) => Some(serde_json::Value::String(s)), graphql_parser::query::Value::List(l) => { let list = l.into_iter().flat_map(graphql_to_json).collect(); - return Some(serde_json::Value::Array(list)); + Some(serde_json::Value::Array(list)) } graphql_parser::query::Value::Object(m) => { let mut map = serde_json::Map::new(); @@ -626,7 +670,7 @@ fn graphql_to_json(val: graphql_parser::query::Value<'_, String>) -> Option Date: Tue, 9 Jun 2026 11:24:35 +0200 Subject: [PATCH 17/20] chore: add some todos --- helper/data-api/src/context.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/helper/data-api/src/context.rs b/helper/data-api/src/context.rs index 5688cd2..a0c68fc 100644 --- a/helper/data-api/src/context.rs +++ b/helper/data-api/src/context.rs @@ -157,6 +157,10 @@ impl GuestDataApi for DataAPIContext { delete, }) = capture_data.capture_stack.pop() { + // TODO + // Problem: This will currently not work when reserving ids for a higher scoped capture, because the reserved ids will be dropped by the time they're needed. + // Solution: Save the reserved ids in the capture data until the capture data is empty. + // if stack_is_empty {clear} else {extend; return extended_vec} let mut reserved_id_map = capture_data .reserve_id_count_per_model .drain() @@ -223,6 +227,7 @@ impl GuestDataApi for DataAPIContext { ); // TODO: set up some kind of mocking so this doesn't break when testing. Same goes for the delete manys and reserve ids. + // TODO: implement chunking so that this doesn't break when we have too many queries. self.request_raw(query, variables)?; } From c7e50487f3038fb929b293db3ecc50d46331e126 Mon Sep 17 00:00:00 2001 From: Robin Walthuis Date: Tue, 16 Jun 2026 14:30:36 +0200 Subject: [PATCH 18/20] fix: save reserved ids in capture data --- helper/data-api/src/context.rs | 89 ++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 43 deletions(-) diff --git a/helper/data-api/src/context.rs b/helper/data-api/src/context.rs index a0c68fc..9aed7e0 100644 --- a/helper/data-api/src/context.rs +++ b/helper/data-api/src/context.rs @@ -22,10 +22,38 @@ pub struct DataAPIContext { #[derive(Debug, Default)] pub struct CaptureData { model_names_of_local_ids: Vec, - reserve_id_count_per_model: HashMap, + reserve_id_count_per_model: HashMap, + reserved_ids: Vec, capture_stack: Vec, } +impl CaptureData { + fn reserve_ids(&mut self, data_api_context: &DataAPIContext) -> Result, String> { + let mut reserved_id_map = self + .reserve_id_count_per_model + .drain() + .map(|entry| data_api_context.reserve_ids(entry)) + .collect::>, String>>()?; + + for model_name in self.model_names_of_local_ids.drain(..) { + let reserved_ids_for_model = reserved_id_map + .get_mut(&model_name) + .ok_or_else(|| format!("ids for model {model_name} were not properly reserved"))? + .pop_front() + .ok_or_else(|| format!("not enough ids were reserved for model {model_name}"))? + as RealId; + + self.reserved_ids.push(reserved_ids_for_model); + } + + if self.capture_stack.is_empty() { + Ok(std::mem::take(&mut self.reserved_ids)) + } else { + Ok(self.reserved_ids.clone()) + } + } +} + #[derive(Default, Debug)] pub struct MassMutateEntries { // TODO: We are probably able to turn these into Vecs because we iterate through them front-to-back anyway @@ -112,28 +140,30 @@ fn generate_delayed_id_response(id: &str, mutation_name: &str) -> JsonString { format!(r#"{{"data": {{"{mutation_name}": {{"id": "{id}"}}}}}}"#) } -fn reserve_ids( - data_api_context: &impl GuestDataApi, - (model_name, id_count): (ModelName, usize), -) -> Result<(ModelName, VecDeque), String> { - let query = r#"mutation ($model: String!, $amount: Int!) { +impl DataAPIContext { + fn reserve_ids( + &self, + (model_name, id_count): (ModelName, u32), + ) -> Result<(ModelName, VecDeque), String> { + let query = r#"mutation ($model: String!, $amount: Int!) { reserveRecords(model: $model, amount: $amount) { ids } }"#; - let variables = format!(r#"{{"model": "{model_name}", "amount": {id_count}}}"#); + let variables = format!(r#"{{"model": "{model_name}", "amount": {id_count}}}"#); - let res = data_api_context.request_raw(query.to_string(), variables)?; + let res = self.request_raw(query.to_string(), variables)?; - let ReserveIdMutationResult { - data: ReserveIdResult { - reserved_ids: ReservedIds { ids }, - }, - } = serde_json::from_str(&res) - .map_err(|_| String::from("could not parse data api result for reserving ids"))?; + let ReserveIdMutationResult { + data: ReserveIdResult { + reserved_ids: ReservedIds { ids }, + }, + } = serde_json::from_str(&res) + .map_err(|_| String::from("could not parse data api result for reserving ids"))?; - Ok((model_name, ids)) + Ok((model_name, ids)) + } } impl GuestDataApi for DataAPIContext { @@ -157,31 +187,7 @@ impl GuestDataApi for DataAPIContext { delete, }) = capture_data.capture_stack.pop() { - // TODO - // Problem: This will currently not work when reserving ids for a higher scoped capture, because the reserved ids will be dropped by the time they're needed. - // Solution: Save the reserved ids in the capture data until the capture data is empty. - // if stack_is_empty {clear} else {extend; return extended_vec} - let mut reserved_id_map = capture_data - .reserve_id_count_per_model - .drain() - .map(|entry| reserve_ids(self, entry)) - .collect::>, String>>()?; - - let reserved_ids = capture_data - .model_names_of_local_ids - .iter() - .map(|model_name| { - Ok(reserved_id_map - .get_mut(model_name) - .ok_or_else(|| { - format!("ids for model {model_name} were not properly reserved") - })? - .pop_front() - .ok_or_else(|| { - format!("not enough ids were reserved for model {model_name}") - })? as RealId) - }) - .collect::, String>>()?; + let reserved_ids = capture_data.reserve_ids(self)?; /* Locally generated ids need to be distinguishable from normal ids after being sent back to the caller and then sent here as for example a related row. @@ -200,9 +206,6 @@ impl GuestDataApi for DataAPIContext { let mut upsert_manys = HashMap::new(); - // If we want to be able to specify a unique by for the upsert we would want to handle them separately, - // but this is left out of scope for now. - for MassMutateEntry { model_name, mut variables, From 0ae3561aca8b9cfef6f69c4e58ba81e3c71c8972 Mon Sep 17 00:00:00 2001 From: Robin Walthuis Date: Thu, 18 Jun 2026 09:17:39 +0200 Subject: [PATCH 19/20] feat: add validation set handling --- helper/data-api/src/context.rs | 90 +++++++++++++++++++++++++++------- 1 file changed, 73 insertions(+), 17 deletions(-) diff --git a/helper/data-api/src/context.rs b/helper/data-api/src/context.rs index 9aed7e0..c8bc6ba 100644 --- a/helper/data-api/src/context.rs +++ b/helper/data-api/src/context.rs @@ -1,4 +1,5 @@ use std::collections::{HashMap, VecDeque}; +use std::fmt::Display; use std::sync::{Arc, Mutex}; use crate::exports::betty_blocks::data_api::data_api::{self, GuestDataApi, JsonString}; @@ -97,6 +98,8 @@ impl MassMutateEntry { } } +// TODO: These don't need to all be public, but might have to be pub(crate) for testing. + #[derive(serde::Serialize, serde::Deserialize)] pub struct ReserveIdMutationResult { data: ReserveIdResult, @@ -116,6 +119,8 @@ pub struct ReservedIds { #[derive(serde::Serialize, serde::Deserialize)] pub struct MutationInput { input: MutationInputVariable, + #[serde(rename = "validationSets")] + validation_sets: Option, } #[derive(serde::Serialize, serde::Deserialize)] @@ -125,6 +130,49 @@ pub struct MutationInputVariable { other_inputs: HashMap, } +#[derive(serde::Serialize, serde::Deserialize)] +pub struct DeleteInput { + id: InternalId, +} + +#[derive(Default)] +pub enum ValidationSets { + Empty, + #[default] + Default, +} + +impl ValidationSets { + fn set_by_mutation_variables( + &mut self, + variables: &serde_json::Map, + ) { + match self { + Self::Default + if variables + .get("validationSets") + .is_some_and(|validation_set| validation_set == "empty") => + { + *self = Self::Empty + } + _ => (), + } + } + + fn as_str(&self) -> &'static str { + match self { + ValidationSets::Default => "default", + ValidationSets::Empty => "empty", + } + } +} + +impl Display for ValidationSets { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.as_str()) + } +} + fn format_mutation(model_name: &str, operation: &DataMutation) -> String { format!( "mutation {{ {}{model_name}(input: $input) {{ id }} }}", @@ -204,28 +252,35 @@ impl GuestDataApi for DataAPIContext { These can be clustered by model, so we can do upsertManyUser but not upsertManyUserAndRole. */ - let mut upsert_manys = HashMap::new(); + let mut upsert_manys: HashMap)> = HashMap::new(); for MassMutateEntry { model_name, mut variables, } in [create, update].into_iter().flatten() { - replace_negative_ids_in_variables(&reserved_ids, &mut variables).unwrap(); + let inputs = variables + .remove("input") + .ok_or_else(|| String::from("mutation did not have an input variable"))?; + + if let serde_json::Value::Object(mut input_variables) = inputs { + replace_negative_ids_in_variables(&reserved_ids, &mut input_variables).unwrap(); + + let (validation_sets, input_vec) = upsert_manys.entry(model_name).or_default(); + + validation_sets.set_by_mutation_variables(&variables); - upsert_manys - .entry(model_name) - .or_insert(Vec::new()) - .push(variables); + input_vec.push(input_variables); + } } - for (model_name, variables) in upsert_manys { + for (model_name, (validation_sets, input_variables)) in upsert_manys { let query = format!("mutation {{ upsertMany{model_name}(input: $input) {{ id }} }}"); let variables = format!( - "{{\"input\": {}}}", - serde_json::to_string(&variables) + "{{\"input\": {}, \"validationSets: {validation_sets}\"}}", + serde_json::to_string(&input_variables) .map_err(|_| String::from("could not format input variables"))? ); @@ -317,10 +372,11 @@ impl GuestDataApi for DataAPIContext { return self.request_raw(query, variables); } - let mutation_name = query_remainder - .split_once(|c: char| c.is_whitespace() || c == '(') - .ok_or_else(|| String::from("query is improperly formatted"))? - .0; + let (mutation_name_with_whitespace, _) = query_remainder + .split_once('(') + .ok_or_else(|| String::from("query is improperly formatted"))?; + + let mutation_name = mutation_name_with_whitespace.trim(); let model_name = mutation_name[6..].to_string(); @@ -331,10 +387,11 @@ impl GuestDataApi for DataAPIContext { }; // It is safe to unwrap the capture stack last_mut here because we checked the length to be non-zero before. This looks a bit wack but is necessary to not mutably borrow capture_data multiple times. - let id: isize = match &query_remainder[0..6] { + let id: isize = match &mutation_name[..6] { "create" => match serde_json::from_str(&variables) { Ok(MutationInput { input: MutationInputVariable { id, .. }, + .. }) => { capture_data .capture_stack @@ -369,6 +426,7 @@ impl GuestDataApi for DataAPIContext { "update" => { let MutationInput { input: MutationInputVariable { id, .. }, + .. } = serde_json::from_str(&variables) .map_err(|_| String::from("could not find id input for update query"))?; @@ -382,9 +440,7 @@ impl GuestDataApi for DataAPIContext { id } "delete" => { - let MutationInput { - input: MutationInputVariable { id, .. }, - } = serde_json::from_str(&variables) + let DeleteInput { id } = serde_json::from_str(&variables) .map_err(|_| String::from("could not find id input for delete query"))?; capture_data From 7051da848c591682009b1c10cb2c336ba017c651 Mon Sep 17 00:00:00 2001 From: Robin Walthuis Date: Thu, 18 Jun 2026 09:24:14 +0200 Subject: [PATCH 20/20] feat: add input chunking --- helper/data-api/src/context.rs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/helper/data-api/src/context.rs b/helper/data-api/src/context.rs index c8bc6ba..8cc0788 100644 --- a/helper/data-api/src/context.rs +++ b/helper/data-api/src/context.rs @@ -9,6 +9,8 @@ type ModelName = String; type InternalId = isize; type RealId = i32; +const CHUNK_SIZE: usize = 100_000; + pub struct DataAPIContext { application_id: String, @@ -278,15 +280,16 @@ impl GuestDataApi for DataAPIContext { let query = format!("mutation {{ upsertMany{model_name}(input: $input) {{ id }} }}"); - let variables = format!( - "{{\"input\": {}, \"validationSets: {validation_sets}\"}}", - serde_json::to_string(&input_variables) - .map_err(|_| String::from("could not format input variables"))? - ); + for input_chunk in input_variables.chunks(CHUNK_SIZE) { + let variables = format!( + "{{\"input\": {}, \"validationSets: {validation_sets}\"}}", + serde_json::to_string(&input_chunk) + .map_err(|_| String::from("could not format input variables"))? + ); - // TODO: set up some kind of mocking so this doesn't break when testing. Same goes for the delete manys and reserve ids. - // TODO: implement chunking so that this doesn't break when we have too many queries. - self.request_raw(query, variables)?; + // TODO: set up some kind of mocking so this doesn't break when testing. Same goes for the delete manys and reserve ids. + self.request_raw(query.clone(), variables)?; + } } let delete_manys = delete