fix(kernel): deterministic form data processing (#4945)

This commit is contained in:
Shreyas 2025-04-08 14:50:04 +05:30 committed by GitHub
parent 3cf286a443
commit 9cc8b68077
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 225 additions and 195 deletions

View file

@ -4098,7 +4098,7 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
[[package]]
name = "relay"
version = "0.1.1"
source = "git+https://github.com/CuriousCorrelation/relay.git#893cec31865dc396a3d351781ec39b7625f59862"
source = "git+https://github.com/CuriousCorrelation/relay.git#cac0d123d0f7ff6971edacf5809c120d5378c25e"
dependencies = [
"bytes",
"curl",
@ -4106,7 +4106,6 @@ dependencies = [
"env_logger",
"http",
"http-serde",
"indexmap 2.8.0",
"infer 0.16.0",
"lazy_static",
"log",

View file

@ -2,8 +2,11 @@ import { pipe } from "fp-ts/function"
import * as O from "fp-ts/Option"
import * as E from "fp-ts/Either"
import * as R from "fp-ts/Record"
import { cloneDeep } from "lodash-es"
import { useSetting } from "~/composables/settings"
import superjson from "superjson"
import type { RelayRequest } from "@hoppscotch/kernel"
const isEncoded = (value: string): boolean =>
@ -67,3 +70,6 @@ export const preProcessRelayRequest = (req: RelayRequest): RelayRequest =>
)
: req
)
export const postProcessRelayRequest = (req: RelayRequest): RelayRequest =>
pipe(cloneDeep(req), (req) => superjson.serialize(req).json)

View file

@ -4,7 +4,10 @@ import { body, relayRequestToNativeAdapter } from "@hoppscotch/kernel"
import * as E from "fp-ts/Either"
import { pipe } from "fp-ts/function"
import axios, { CancelTokenSource } from "axios"
import { preProcessRelayRequest } from "~/helpers/functional/preprocess"
import {
postProcessRelayRequest,
preProcessRelayRequest,
} from "~/helpers/functional/process-request"
import {
RelayRequest,
RelayResponse,
@ -148,8 +151,13 @@ export class AgentKernelInterceptorService
},
}
const nativeRequest = await relayRequestToNativeAdapter(
effectiveRequestWithUserAgent
)
const postProcessedRequest = postProcessRelayRequest(nativeRequest)
const [nonceB16, encryptedReq] = await this.store.encryptRequest(
await relayRequestToNativeAdapter(effectiveRequestWithUserAgent),
postProcessedRequest,
reqID
)

View file

@ -2,7 +2,6 @@ import { Service } from "dioc"
import { ref } from "vue"
import * as E from "fp-ts/Either"
import axios from "axios"
import superjson from "superjson"
import { Store } from "~/kernel/store"
import type { PluginRequest, PluginResponse } from "@hoppscotch/kernel"
import { x25519 } from "@noble/curves/ed25519"
@ -275,9 +274,7 @@ export class KernelInterceptorAgentStore extends Service {
request: PluginRequest,
reqID: number
): Promise<[string, ArrayBuffer]> {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const { json, meta: _ } = superjson.serialize({ ...request, id: reqID })
const reqJSON = JSON.stringify(json)
const reqJSON = JSON.stringify({ ...request, id: reqID })
const reqJSONBytes = new TextEncoder().encode(reqJSON)
const nonce = window.crypto.getRandomValues(new Uint8Array(12))
const nonceB16 = base16.encode(nonce).toLowerCase()

View file

@ -12,7 +12,7 @@ import type {
} from "~/services/kernel-interceptor.service"
import { getI18n } from "~/modules/i18n"
import { preProcessRelayRequest } from "~/helpers/functional/preprocess"
import { preProcessRelayRequest } from "~/helpers/functional/process-request"
import InterceptorsErrorPlaceholder from "~/components/interceptors/ErrorPlaceholder.vue"

View file

@ -7,7 +7,7 @@ import SettingsExtensionSubtitle from "~/components/settings/ExtensionSubtitle.v
import * as E from "fp-ts/Either"
import { getI18n } from "~/modules/i18n"
import { until } from "@vueuse/core"
import { preProcessRelayRequest } from "~/helpers/functional/preprocess"
import { preProcessRelayRequest } from "~/helpers/functional/process-request"
import { browserIsChrome, browserIsFirefox } from "~/helpers/utils/userAgent"
import type {
KernelInterceptor,

View file

@ -2,8 +2,16 @@ import { markRaw } from "vue"
import * as E from "fp-ts/Either"
import { pipe } from "fp-ts/function"
import { getI18n } from "~/modules/i18n"
import { preProcessRelayRequest } from "~/helpers/functional/preprocess"
import type { RelayCapabilities, RelayRequest } from "@hoppscotch/kernel"
import {
postProcessRelayRequest,
preProcessRelayRequest,
} from "~/helpers/functional/process-request"
import {
relayRequestToNativeAdapter,
type RelayCapabilities,
type RelayRequest,
type RelayResponse,
} from "@hoppscotch/kernel"
import { Relay } from "~/kernel/relay"
import { Service } from "dioc"
import type {
@ -70,124 +78,150 @@ export class NativeKernelInterceptorService
public execute(
request: RelayRequest
): ExecutionResult<KernelInterceptorError> {
const effectiveRequest = this.store.completeRequest(
preProcessRelayRequest(request)
)
const relevantCookies = this.cookieJar.getCookiesForURL(
new URL(effectiveRequest.url!)
)
if (relevantCookies.length > 0) {
effectiveRequest.headers!["Cookie"] = relevantCookies
.map((cookie) => `${cookie.name!}=${cookie.value!}`)
.join(";")
}
const existingUserAgentHeader = Object.keys(
effectiveRequest.headers || {}
).find((header) => header.toLowerCase() === "user-agent")
// A temporary workaround to add a User-Agent header to the request
// This will be removed once the kernel/relay is updated to add User-Agent header by default
const effectiveRequestWithUserAgent = {
...effectiveRequest,
headers: {
...effectiveRequest.headers,
"User-Agent": existingUserAgentHeader
? effectiveRequest.headers[existingUserAgentHeader]
: "HoppscotchKernel/0.1.0",
},
}
const relayExecution = Relay.execute(effectiveRequestWithUserAgent)
const response = pipe(relayExecution.response, (promise) =>
promise.then((either) =>
pipe(
either,
E.mapLeft((error): KernelInterceptorError => {
const humanMessage = {
heading: (t: ReturnType<typeof getI18n>) => {
switch (error.kind) {
case "network":
return t("error.network.heading")
case "timeout":
return t("error.timeout.heading")
case "certificate":
return t("error.certificate.heading")
case "auth":
return t("error.auth.heading")
case "proxy":
return t("error.proxy.heading")
case "parse":
return t("error.parse.heading")
case "version":
return t("error.version.heading")
case "abort":
return t("error.aborted.heading")
default:
return t("error.unknown.heading")
}
},
description: (t: ReturnType<typeof getI18n>) => {
switch (error.kind) {
case "network":
return t("error.network.description", {
message: error.message,
cause: error.cause ?? t("error.unknown.cause"),
})
case "timeout":
return t("error.timeout.description", {
message: error.message,
phase: error.phase ?? t("error.unknown.phase"),
})
case "certificate":
return t("error.certificate.description", {
message: error.message,
cause: error.cause ?? t("error.unknown.cause"),
})
case "auth":
return t("error.auth.description", {
message: error.message,
cause: error.cause ?? t("error.unknown.cause"),
})
case "proxy":
return t("error.proxy.description", {
message: error.message,
cause: error.cause ?? t("error.unknown.cause"),
})
case "parse":
return t("error.parse.description", {
message: error.message,
cause: error.cause ?? t("error.unknown.cause"),
})
case "version":
return t("error.version.description", {
message: error.message,
cause: error.cause ?? t("error.unknown.cause"),
})
case "abort":
return t("error.aborted.description", {
message: error.message,
})
default:
return t("error.unknown.description")
}
},
}
return {
humanMessage,
error,
component: InterceptorsErrorPlaceholder,
}
})
)
)
)
let relayExecution: { cancel: () => Promise<void> } | null = null
return {
cancel: relayExecution.cancel,
response,
cancel: async () => {
if (relayExecution) {
await relayExecution.cancel()
}
},
response: pipe(
this.executeRequest(request, (execution) => {
relayExecution = execution
}),
(promise) =>
promise.then((either) =>
pipe(
either,
E.mapLeft((error): KernelInterceptorError => {
const humanMessage = {
heading: (t: ReturnType<typeof getI18n>) => {
switch (error.kind) {
case "network":
return t("error.network.heading")
case "timeout":
return t("error.timeout.heading")
case "certificate":
return t("error.certificate.heading")
case "auth":
return t("error.auth.heading")
case "proxy":
return t("error.proxy.heading")
case "parse":
return t("error.parse.heading")
case "version":
return t("error.version.heading")
case "abort":
return t("error.aborted.heading")
default:
return t("error.unknown.heading")
}
},
description: (t: ReturnType<typeof getI18n>) => {
switch (error.kind) {
case "network":
return t("error.network.description", {
message: error.message,
cause: error.cause ?? t("error.unknown.cause"),
})
case "timeout":
return t("error.timeout.description", {
message: error.message,
phase: error.phase ?? t("error.unknown.phase"),
})
case "certificate":
return t("error.certificate.description", {
message: error.message,
cause: error.cause ?? t("error.unknown.cause"),
})
case "auth":
return t("error.auth.description", {
message: error.message,
cause: error.cause ?? t("error.unknown.cause"),
})
case "proxy":
return t("error.proxy.description", {
message: error.message,
cause: error.cause ?? t("error.unknown.cause"),
})
case "parse":
return t("error.parse.description", {
message: error.message,
cause: error.cause ?? t("error.unknown.cause"),
})
case "version":
return t("error.version.description", {
message: error.message,
cause: error.cause ?? t("error.unknown.cause"),
})
case "abort":
return t("error.aborted.description", {
message: error.message,
})
default:
return t("error.unknown.description")
}
},
}
return {
humanMessage,
error,
component: InterceptorsErrorPlaceholder,
}
})
)
)
),
}
}
private async executeRequest(
request: RelayRequest,
setRelayExecution: (execution: { cancel: () => Promise<void> }) => void
): Promise<E.Either<any, RelayResponse>> {
try {
const effectiveRequest = this.store.completeRequest(
preProcessRelayRequest(request)
)
const relevantCookies = this.cookieJar.getCookiesForURL(
new URL(effectiveRequest.url!)
)
if (relevantCookies.length > 0) {
effectiveRequest.headers!["Cookie"] = relevantCookies
.map((cookie) => `${cookie.name!}=${cookie.value!}`)
.join(";")
}
const existingUserAgentHeader = Object.keys(
effectiveRequest.headers || {}
).find((header) => header.toLowerCase() === "user-agent")
// A temporary workaround to add a User-Agent header to the request
// This will be removed once the kernel/relay is updated to add User-Agent header by default
const effectiveRequestWithUserAgent = {
...effectiveRequest,
headers: {
...effectiveRequest.headers,
"User-Agent": existingUserAgentHeader
? effectiveRequest.headers[existingUserAgentHeader]
: "HoppscotchKernel/0.1.0",
},
}
const nativeRequest = await relayRequestToNativeAdapter(
effectiveRequestWithUserAgent
)
const postProcessedRequest = postProcessRelayRequest(nativeRequest)
const relayExecution = Relay.execute(postProcessedRequest)
setRelayExecution(relayExecution)
return await relayExecution.response
} catch (e) {
return E.left(e)
}
}
}

View file

@ -21,7 +21,7 @@ import { pipe } from "fp-ts/function"
import { getI18n } from "~/modules/i18n"
import { v4 } from "uuid"
import { preProcessRelayRequest } from "~/helpers/functional/preprocess"
import { preProcessRelayRequest } from "~/helpers/functional/process-request"
import { parseBytesToJSON } from "~/helpers/functional/json"
import { decodeB64StringToArrayBuffer } from "~/helpers/utils/b64"

View file

@ -188,7 +188,7 @@ checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
dependencies = [
"cfg-if",
"crossbeam-utils",
"hashbrown 0.14.5",
"hashbrown",
"lock_api",
"once_cell",
"parking_lot_core",
@ -238,12 +238,6 @@ dependencies = [
"log",
]
[[package]]
name = "equivalent"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
[[package]]
name = "fnv"
version = "1.0.7"
@ -298,12 +292,6 @@ version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
[[package]]
name = "hashbrown"
version = "0.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
[[package]]
name = "heck"
version = "0.5.0"
@ -476,17 +464,6 @@ dependencies = [
"icu_properties",
]
[[package]]
name = "indexmap"
version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058"
dependencies = [
"equivalent",
"hashbrown 0.15.2",
"serde",
]
[[package]]
name = "infer"
version = "0.16.0"
@ -753,7 +730,6 @@ dependencies = [
"env_logger",
"http",
"http-serde",
"indexmap",
"infer",
"lazy_static",
"log",

View file

@ -30,4 +30,3 @@ strum = { version = "0.26.3", features = ["derive"] }
bytes = { version = "1.9.0", features = ["serde"] }
mime = "0.3.17"
url = "2.5.4"
indexmap = { version = "2.8.0", features = ["serde"] }

View file

@ -1,6 +1,5 @@
use curl::easy::Easy;
use http::HeaderName;
use indexmap::IndexMap;
use std::{collections::HashMap, path::Path};
use crate::{
@ -201,7 +200,7 @@ impl<'a> ContentHandler<'a> {
fn set_form_content(
&mut self,
content: &IndexMap<String, Vec<FormValue>>,
content: &Vec<(String, Vec<FormValue>)>,
media_type: &MediaType,
) -> Result<()> {
/* TODO: Look into reintroducing this when auth handling is done by kernel */
@ -277,7 +276,7 @@ impl<'a> ContentHandler<'a> {
fn set_multipart_content(
&mut self,
content: &IndexMap<String, Vec<FormValue>>,
content: &Vec<(String, Vec<FormValue>)>,
media_type: &MediaType,
) -> Result<()> {
self.set_form_content(content, media_type)

View file

@ -2,7 +2,6 @@ use std::collections::HashMap;
use bytes::Bytes;
use http::{Method, StatusCode, Version};
use indexmap::IndexMap;
use serde::{Deserialize, Serialize};
use strum::{Display, EnumString};
use time::OffsetDateTime;
@ -59,7 +58,7 @@ pub enum FormValue {
},
}
pub type FormData = IndexMap<String, Vec<FormValue>>;
pub type FormData = Vec<(String, Vec<FormValue>)>;
#[derive(Debug, Serialize, Deserialize, Clone)]
#[serde(tag = "kind", rename_all = "camelCase")]

View file

@ -2888,7 +2888,7 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
[[package]]
name = "relay"
version = "0.1.1"
source = "git+https://github.com/CuriousCorrelation/relay.git#893cec31865dc396a3d351781ec39b7625f59862"
source = "git+https://github.com/CuriousCorrelation/relay.git#cac0d123d0f7ff6971edacf5809c120d5378c25e"
dependencies = [
"bytes",
"curl",
@ -2896,7 +2896,6 @@ dependencies = [
"env_logger",
"http",
"http-serde",
"indexmap 2.8.0",
"infer 0.16.0",
"lazy_static",
"log",

View file

@ -10,7 +10,7 @@ export type FormDataValue = {
contentType: string;
data: Uint8Array;
};
export type FormData = Map<string, FormDataValue[]>;
export type FormData = [string, FormDataValue[]][];
export declare enum MediaType {
TEXT_PLAIN = "text/plain",
TEXT_HTML = "text/html",

File diff suppressed because one or more lines are too long

View file

@ -82,7 +82,7 @@ export type FormDataValue =
| { kind: "text"; value: string }
| { kind: "file"; filename: string; contentType: string; data: Uint8Array }
export type FormData = Map<string, FormDataValue[]>
export type FormData = [string, FormDataValue[]][]
export enum MediaType {
TEXT_PLAIN = "text/plain",

View file

@ -3956,7 +3956,7 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
[[package]]
name = "relay"
version = "0.1.1"
source = "git+https://github.com/CuriousCorrelation/relay.git#893cec31865dc396a3d351781ec39b7625f59862"
source = "git+https://github.com/CuriousCorrelation/relay.git#cac0d123d0f7ff6971edacf5809c120d5378c25e"
dependencies = [
"bytes",
"curl",
@ -3964,7 +3964,6 @@ dependencies = [
"env_logger",
"http",
"http-serde",
"indexmap 2.8.0",
"infer",
"lazy_static",
"log",
@ -5082,7 +5081,7 @@ dependencies = [
[[package]]
name = "tauri-plugin-relay"
version = "0.1.0"
source = "git+https://github.com/CuriousCorrelation/tauri-plugin-relay#fee58601fb4b0c129b5a9d87aaa0f9f87c26cd09"
source = "git+https://github.com/CuriousCorrelation/tauri-plugin-relay#68d6b2532c900b4be24a038c49eec4794e990a3d"
dependencies = [
"relay",
"serde",

View file

@ -590,11 +590,26 @@ export const content = {
})
}
// Helper function to convert standard `FormData` to `Map<string, FormDataValue[]>`
// This is mainly a crossplatform thing, once there's an equivalent and easy to impl `FormData` type for Rust,
// we can consider removing this.
const makeFormDataSerializable = async (formData: FormData): Promise<Map<string, FormDataValue[]>> => {
const result = new Map<string, FormDataValue[]>()
/**
* Helper function to convert standard `FormData` to array of arrays `[string, FormDataValue[]][]`
*
* This implementation uses a Map to maintain insertion order of form fields,
* required for certain multipart/form-data requests where field order matters.
*
* JavaScript Maps maintain insertion order (ECMAScript 2015+) unlike plain objects
* before ES2015 ("own properties") where property enumeration order was not guaranteed,
* See: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Map#description
* > Although the keys of an ordinary Object are ordered now,
* > this was not always the case, and the order is complex.
* > As a result, it's best not to rely on property order.
*
* This preserves the original field order as per RFC 7578 section 5.2.
* See: https://datatracker.ietf.org/doc/html/rfc7578#section-5.2
* > Form processors given forms with a well-defined ordering SHOULD send back results in order.
*/
const makeFormDataSerializable = async (formData: FormData): Promise<[string, FormDataValue[]][]> => {
const m = new Map<string, FormDataValue[]>()
// @ts-expect-error: `formData.entries` does exist but isn't visible,
// see `"lib": ["ESNext", "DOM"],` in `tsconfig.json`
for (const [key, value] of formData.entries()) {
@ -607,20 +622,26 @@ const makeFormDataSerializable = async (formData: FormData): Promise<Map<string,
data: new Uint8Array(buffer)
}
const existingValues = result.get(key) || []
result.set(key, [...existingValues, fileEntry])
if (m.has(key)) {
m.get(key)!.push(fileEntry)
} else {
m.set(key, [fileEntry])
}
} else {
const textEntry: FormDataValue = {
kind: "text",
value: value.toString()
}
const existingValues = result.get(key) || []
result.set(key, [...existingValues, textEntry])
if (m.has(key)) {
m.get(key)!.push(textEntry)
} else {
m.set(key, [textEntry])
}
}
}
return result
return Array.from(m.entries())
}
// Helper function to adapt a relay request to work with the plugin
@ -630,22 +651,16 @@ export const relayRequestToNativeAdapter = async (request: RelayRequest): Promis
if (adaptedRequest.content?.kind === "multipart" && adaptedRequest.content.content instanceof FormData) {
const serializableFormData = await makeFormDataSerializable(adaptedRequest.content.content);
// Replace with the converted form data
// SAFETY: Type assertion is necessary here because the plugin system expects
// types similar to Map<string, FormDataValue[]> instead of FormData.
// Then convert the `Map` to simpler nested object structure for better compatibility
// `Maps` it seems like are serialized differently across platforms and serialization libraries,
// while objects tend to maintain more consistent behavior by the sheer ubiquity of it.
const convertedContent: Record<string, FormDataValue[]> = {};
for (const [key, values] of serializableFormData.entries()) {
convertedContent[key] = Array.isArray(values) ? values : [values];
}
adaptedRequest.content = {
...adaptedRequest.content,
// Replace with the converted form data
// SAFETY: Type assertion is necessary here because the plugin system expects
// types similar to Map<string, FormDataValue[]> instead of FormData.
// Then convert the `Map` to simpler nested `Array` of `Array` structure for better compatibility
// `Maps` it seems like are serialized differently across platforms and serialization libraries,
// while `Array` of `Array` tend to maintain more consistent behavior by the sheer ubiquity of it.
// @ts-expect-error: This is intentional to work around SuperJSON serialization
content: convertedContent
content: serializableFormData
};
}

View file

@ -1181,7 +1181,7 @@ importers:
dependencies:
'@hoppscotch/plugin-relay':
specifier: github:CuriousCorrelation/tauri-plugin-relay
version: '@CuriousCorrelation/plugin-relay@https://codeload.github.com/CuriousCorrelation/tauri-plugin-relay/tar.gz/fee58601fb4b0c129b5a9d87aaa0f9f87c26cd09'
version: '@CuriousCorrelation/plugin-relay@https://codeload.github.com/CuriousCorrelation/tauri-plugin-relay/tar.gz/68d6b2532c900b4be24a038c49eec4794e990a3d'
'@tauri-apps/api':
specifier: 2.1.1
version: 2.1.1
@ -1810,8 +1810,8 @@ packages:
resolution: {tarball: https://codeload.github.com/CuriousCorrelation/tauri-plugin-appload/tar.gz/1c2e8b19db7f1b6af6d00abb907f15cdc2017298}
version: 0.1.0
'@CuriousCorrelation/plugin-relay@https://codeload.github.com/CuriousCorrelation/tauri-plugin-relay/tar.gz/fee58601fb4b0c129b5a9d87aaa0f9f87c26cd09':
resolution: {tarball: https://codeload.github.com/CuriousCorrelation/tauri-plugin-relay/tar.gz/fee58601fb4b0c129b5a9d87aaa0f9f87c26cd09}
'@CuriousCorrelation/plugin-relay@https://codeload.github.com/CuriousCorrelation/tauri-plugin-relay/tar.gz/68d6b2532c900b4be24a038c49eec4794e990a3d':
resolution: {tarball: https://codeload.github.com/CuriousCorrelation/tauri-plugin-relay/tar.gz/68d6b2532c900b4be24a038c49eec4794e990a3d}
version: 0.1.0
'@alloc/quick-lru@5.2.0':
@ -13208,7 +13208,7 @@ snapshots:
dependencies:
'@tauri-apps/api': 2.1.1
'@CuriousCorrelation/plugin-relay@https://codeload.github.com/CuriousCorrelation/tauri-plugin-relay/tar.gz/fee58601fb4b0c129b5a9d87aaa0f9f87c26cd09':
'@CuriousCorrelation/plugin-relay@https://codeload.github.com/CuriousCorrelation/tauri-plugin-relay/tar.gz/68d6b2532c900b4be24a038c49eec4794e990a3d':
dependencies:
'@tauri-apps/api': 2.1.1