diff --git a/shortcuts/base/base_execute_test.go b/shortcuts/base/base_execute_test.go index 371400729a..20cebf1610 100644 --- a/shortcuts/base/base_execute_test.go +++ b/shortcuts/base/base_execute_test.go @@ -2028,7 +2028,7 @@ func TestBaseRecordExecuteReadCreateDelete(t *testing.T) { factory, stdout, reg := newExecuteFactory(t) reg.Register(&httpmock.Stub{ Method: "GET", - URL: "field_id=Name&field_id=Age&limit=2&offset=0", + URL: "field_id=Name&field_id=Age&field_id=Formula&limit=2&offset=0", Body: map[string]interface{}{ "code": 0, "data": map[string]interface{}{ @@ -2044,11 +2044,15 @@ func TestBaseRecordExecuteReadCreateDelete(t *testing.T) { "record_scope": "all_records", "field_scope": "selected_fields", }, - "ignored_fields": []interface{}{"Formula"}, + "ignored_fields": []interface{}{map[string]interface{}{ + "id": "fld_formula", + "name": "Formula", + "reason": "UNSUPPORTED: formula field cannot be read through OpenAPI because this base uses an old schema version without backend formula computation.", + }}, }, }, }) - if err := runShortcut(t, BaseRecordList, []string{"+record-list", "--base-token", "app_x", "--table-id", "tbl_x", "--limit", "2", "--field-id", "Name", "--field-id", "Age"}, factory, stdout); err != nil { + if err := runShortcut(t, BaseRecordList, []string{"+record-list", "--base-token", "app_x", "--table-id", "tbl_x", "--limit", "2", "--field-id", "Name", "--field-id", "Age", "--field-id", "Formula"}, factory, stdout); err != nil { t.Fatalf("err=%v", err) } got := stdout.String() @@ -2057,7 +2061,7 @@ func TestBaseRecordExecuteReadCreateDelete(t *testing.T) { "| _record_id | Name | Age |", "| rec_1 | Alice | 18 |", "Meta: count=2; has_more=false; record_scope=all_records; field_scope=selected_fields; ignored_fields=1", - "Ignored fields: Formula", + `Ignored fields: {"id":"fld_formula","name":"Formula","reason":"UNSUPPORTED: formula field cannot be read through OpenAPI because this base uses an old schema version without backend formula computation."}`, } { if !strings.Contains(got, want) { t.Fatalf("stdout missing %q:\n%s", want, got) @@ -2814,11 +2818,15 @@ func TestBaseRecordExecuteReadCreateDelete(t *testing.T) { Body: map[string]interface{}{ "code": 0, "data": map[string]interface{}{ - "ignored_fields": []interface{}{"Formula"}, + "ignored_fields": []interface{}{map[string]interface{}{ + "id": "fld_formula", + "name": "Formula", + "reason": "READONLY: formula field cannot be written through OpenAPI.", + }}, }, }, }) - if err := runShortcut(t, BaseRecordBatchUpdate, []string{"+record-batch-update", "--base-token", "app_x", "--table-id", "tbl_x", "--json", `{"update_records":{"rec_1":{"Status":["Done"]}}}`}, factory, stdout); err != nil { + if err := runShortcut(t, BaseRecordBatchUpdate, []string{"+record-batch-update", "--base-token", "app_x", "--table-id", "tbl_x", "--json", `{"update_records":{"rec_1":{"Status":["Done"],"Formula":"ignored"}}}`}, factory, stdout); err != nil { t.Fatalf("err=%v", err) } if got := stdout.String(); !strings.Contains(got, `"ignored_fields"`) || !strings.Contains(got, `"Formula"`) { diff --git a/shortcuts/base/base_shortcuts_test.go b/shortcuts/base/base_shortcuts_test.go index 96de23a4a6..956bece447 100644 --- a/shortcuts/base/base_shortcuts_test.go +++ b/shortcuts/base/base_shortcuts_test.go @@ -360,8 +360,9 @@ func TestBaseRecordReadHelpGuidesAgents(t *testing.T) { "view ID or name; omit for reading all table records, or set to read a user-specified or temporary filtered/sorted view", `filter JSON object or @file`, `sort JSON array or @file`, - "pagination size, range 1-200", - "output format: markdown (default) | json", + "maximum records to return; range 1-200, or 1-2000 for ndjson", + "ndjson typed artifact (preferred for analysis)", + "preferred analysis output: relative .ndjson output path", }, wantTips: []string{ "lark-cli base +record-list --base-token --table-id --limit 50", @@ -369,7 +370,8 @@ func TestBaseRecordReadHelpGuidesAgents(t *testing.T) { "Text equality filter", "Option intersection filter", "Query priority", - "Default output is markdown", + "Example for analysis", + "prefer --output ./records.ndjson --minimal-stdout", "Use --field-id repeatedly to keep output small", }, }, @@ -382,7 +384,8 @@ func TestBaseRecordReadHelpGuidesAgents(t *testing.T) { "field ID or name to search", `filter JSON object or @file`, `sort JSON array or @file`, - "output format: markdown (default) | json", + "ndjson typed artifact (preferred for analysis)", + "preferred analysis output: relative .ndjson output path", }, wantTips: []string{ "Example: lark-cli base +record-search", @@ -390,7 +393,8 @@ func TestBaseRecordReadHelpGuidesAgents(t *testing.T) { "Text equality filter", "Query priority", "Use --json only when you need to pass the full search body directly", - "Default output is markdown", + "Example for analysis", + "prefer --output ./records.ndjson --minimal-stdout", }, }, { @@ -399,15 +403,16 @@ func TestBaseRecordReadHelpGuidesAgents(t *testing.T) { wantHelp: []string{ "record ID (repeatable)", "field ID or name to project; repeat to keep only needed columns", - "output format: markdown (default) | json", + "ndjson typed artifact (preferred for analysis)", + "preferred analysis output: relative .ndjson output path", }, wantTips: []string{ "lark-cli base +record-get --base-token --table-id --record-id ", "lark-cli base +record-get --base-token --table-id --record-id rec_001 --record-id rec_002 --field-id Name --field-id Status", - "Default output is markdown", + "Example for analysis input", + "prefer --output ./records.ndjson --minimal-stdout", "projection boundary", "record_id is already known", - "lark-base record read SOP", }, }, } @@ -448,8 +453,8 @@ func TestBasePaginationHelpShowsDefaults(t *testing.T) { {name: "table list", shortcut: BaseTableList, flag: "limit", defaultVal: "50", help: "pagination size, range 1-100"}, {name: "field list", shortcut: BaseFieldList, flag: "limit", defaultVal: "100", help: "pagination size, range 1-200"}, {name: "field search options", shortcut: BaseFieldSearchOptions, flag: "limit", defaultVal: "30", help: "pagination size, range 1-200"}, - {name: "record list", shortcut: BaseRecordList, flag: "limit", defaultVal: "100", help: "pagination size, range 1-200"}, - {name: "record search", shortcut: BaseRecordSearch, flag: "limit", defaultVal: "10", help: "pagination size, range 1-200"}, + {name: "record list", shortcut: BaseRecordList, flag: "limit", defaultVal: "100", help: "maximum records to return; range 1-200, or 1-2000 for ndjson"}, + {name: "record search", shortcut: BaseRecordSearch, flag: "limit", defaultVal: "10", help: "maximum records to return; range 1-200, or 1-2000 for ndjson"}, {name: "view list", shortcut: BaseViewList, flag: "limit", defaultVal: "100", help: "pagination size, range 1-200"}, {name: "form list", shortcut: BaseFormsList, flag: "page-size", defaultVal: "100", help: "page size per request, range 1-100"}, {name: "workflow list", shortcut: BaseWorkflowList, flag: "page-size", defaultVal: "100", help: "page size per request, range 1-100"}, @@ -932,11 +937,13 @@ func TestBaseRecordWriteHelpGuidesAgents(t *testing.T) { `{"Parent Link":[{"id":"rec_xxx"}]}`, "do not look for parent_record_id or a separate child-record API", "CellValue happy path: text/phone/url", - "select (multiple=false) -> \"Todo\"", - "select (multiple=true) -> [\"Tag A\",\"Tag B\"]", - "datetime -> \"2026-03-24 10:00:00\"", + "select -> [\"Todo\"] or [\"Tag A\",\"Tag B\"]", + "when multiple=false, the array can contain only one option", + "datetime -> \"2026-03-24 10:00\"", "checkbox -> true/false", `ID-based CellValue: user/group/link fields use arrays like [{"id":"ou_xxx"}]`, + "User and group fields always use arrays", + "when multiple=false, the array can contain only one item", `location uses {"lng":116.397428,"lat":39.90923}`, "Do not guess user/chat/linked-record IDs or location coordinates", "lark-base-cell-value.md", diff --git a/shortcuts/base/record_export.go b/shortcuts/base/record_export.go new file mode 100644 index 0000000000..e8d51dedd2 --- /dev/null +++ b/shortcuts/base/record_export.go @@ -0,0 +1,507 @@ +// Copyright (c) 2026 Lark Technologies Pte. Ltd. +// SPDX-License-Identifier: MIT + +package base + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "io/fs" + "path/filepath" + "strings" + "time" + + "github.com/larksuite/cli/errs" + "github.com/larksuite/cli/extension/fileio" + "github.com/larksuite/cli/internal/output" + "github.com/larksuite/cli/shortcuts/base/recordexport" + "github.com/larksuite/cli/shortcuts/common" +) + +const ( + maxInlineRecordReadLimit = 200 + maxNDJSONRecordReadLimit = 2000 + recordAnalysisOutputTip = "For analysis, parsing, comparison, or reusable local input, prefer --output ./records.ndjson --minimal-stdout; use inline markdown/json for small results intended for immediate display." +) + +var recordExportNow = time.Now + +func recordOutputFlag() common.Flag { + return common.Flag{ + Name: "output", + Desc: "preferred analysis output: relative .ndjson output path; implies --format ndjson when format is omitted", + } +} + +func recordMinimalStdoutFlag() common.Flag { + return common.Flag{ + Name: "minimal-stdout", Type: "bool", + Desc: "for ndjson, print only artifact paths, record file size, records_count, and has_more", + } +} + +func recordJQRecordsFlag() common.Flag { + return common.Flag{ + Name: "jq-records", + Desc: "for ndjson, run a jq expression once against the exported records array and print the result; artifact files remain unchanged", + } +} + +func recordOverwriteFlag() common.Flag { + return common.Flag{ + Name: "overwrite", Type: "bool", + Desc: "replace an existing ndjson artifact and its manifest", + } +} + +func normalizeRecordReadOutput(_ context.Context, flags *common.FlagContext) error { + if strings.TrimSpace(flags.Str("output")) == "" { + return nil + } + if flags.Changed("format") && flags.Str("format") != recordexport.FormatNDJSON { + return errs.NewValidationError( + errs.SubtypeInvalidArgument, + "--output writes an ndjson artifact and conflicts with --format %s", + flags.Str("format"), + ). + WithParam("--output"). + WithParams( + errs.InvalidParam{Name: "--output", Reason: "requires ndjson"}, + errs.InvalidParam{Name: "--format", Reason: "conflicts with --output"}, + ). + WithHint("Remove --format or set --format ndjson.") + } + if !flags.Changed("format") { + return flags.SetCanonicalFrom("output", "format", recordexport.FormatNDJSON) + } + return nil +} + +func validateRecordExportFlags(runtime *common.RuntimeContext) error { + format := runtime.Str("format") + outputPath := strings.TrimSpace(runtime.Str("output")) + jqRecords := strings.TrimSpace(runtime.Str("jq-records")) + if format != recordexport.FormatNDJSON { + switch { + case outputPath != "": + return baseFlagErrorf("--output requires --format ndjson") + case runtime.Bool("minimal-stdout"): + return baseFlagErrorf("--minimal-stdout requires --format ndjson") + case jqRecords != "": + return baseFlagErrorf("--jq-records requires --format ndjson") + case runtime.Bool("overwrite"): + return baseFlagErrorf("--overwrite requires --format ndjson") + } + return nil + } + if jqRecords != "" { + switch { + case runtime.JqExpr != "": + return baseFlagErrorf("--jq-records and --jq are mutually exclusive") + case runtime.Bool("minimal-stdout"): + return baseFlagErrorf("--jq-records and --minimal-stdout are mutually exclusive") + } + if err := output.ValidateJqExpression(jqRecords); err != nil { + return err + } + } + if outputPath != "" { + if filepath.Ext(outputPath) != ".ndjson" { + return errs.NewValidationError(errs.SubtypeInvalidArgument, "--output must end with .ndjson"). + WithParam("--output"). + WithHint("Use a path such as ./exports/records.ndjson.") + } + fio := runtime.FileIO() + if fio == nil { + return baseMissingFileIOError("record export requires a file I/O provider") + } + if _, err := fio.ResolvePath(outputPath); err != nil { + return baseSaveError(err) + } + } + return nil +} + +// validateRecordReadLimit intentionally runs after format normalization so an +// inferred ndjson format receives the 2000-row bound instead of the inline +// 200-row bound. +func validateRecordReadLimit(runtime *common.RuntimeContext, defaultLimit int) error { + maximum := maxInlineRecordReadLimit + if runtime.Str("format") == recordexport.FormatNDJSON { + maximum = maxNDJSONRecordReadLimit + } + _, err := common.ValidatePageSizeTyped(runtime, "limit", defaultLimit, 1, maximum) + return err +} + +type recordExportAccumulator struct { + dataset recordexport.Dataset + rev *int64 + initialized bool + pageCount int + hasMore bool + queryContext map[string]any + ignoredFields []recordexport.IgnoredField + recordNotFound []string +} + +func (a *recordExportAccumulator) append(page recordexport.Page) error { + if !a.initialized { + a.dataset = page.Dataset + a.rev = page.Rev + a.queryContext = page.QueryContext + a.initialized = true + } else if err := a.dataset.AppendPage(page); err != nil { + return err + } + a.pageCount++ + a.hasMore = page.HasMore + a.ignoredFields = appendUniqueIgnoredFields(a.ignoredFields, page.IgnoredFields) + a.recordNotFound = appendUniqueStrings(a.recordNotFound, page.RecordNotFound) + return nil +} + +func parseRecordExportPage(data map[string]any) (recordexport.Page, error) { + page, err := recordexport.ParseMatrix(data) + if err == nil { + return page, nil + } + return recordexport.Page{}, errs.NewInternalError( + errs.SubtypeInvalidResponse, "cannot export record matrix: %v", err, + ).WithCause(err) +} + +func appendRecordExportPage(accumulator *recordExportAccumulator, page recordexport.Page) error { + if err := accumulator.append(page); err != nil { + var schemaChanged *recordexport.SchemaChangedError + if errors.As(err, &schemaChanged) { + return errs.NewValidationError( + errs.SubtypeFailedPrecondition, + "table schema changed during download; this request failed", + ). + WithHint("Retry the request so every page uses one consistent schema."). + WithCause(err) + } + return errs.NewInternalError(errs.SubtypeInvalidResponse, "cannot append record page: %v", err).WithCause(err) + } + return nil +} + +func executeRecordListNDJSON( + runtime *common.RuntimeContext, + baseParams map[string]any, + startOffset int, + requestedLimit int, +) error { + accumulator := &recordExportAccumulator{} + currentOffset := startOffset + remaining := requestedLimit + for remaining > 0 { + pageLimit := min(remaining, maxInlineRecordReadLimit) + params := cloneMap(baseParams) + params["offset"] = currentOffset + params["limit"] = pageLimit + data, err := baseV3Call(runtime, "GET", baseV3Path( + "bases", runtime.Str("base-token"), "tables", baseTableID(runtime), "records", + ), params, nil) + if err != nil { + return err + } + page, err := parseRecordExportPage(data) + if err != nil { + return err + } + if len(page.Dataset.Records) > pageLimit { + return errs.NewInternalError( + errs.SubtypeInvalidResponse, + "record API returned %d rows for page limit %d", len(page.Dataset.Records), pageLimit, + ) + } + if err := appendRecordExportPage(accumulator, page); err != nil { + return err + } + count := len(page.Dataset.Records) + remaining -= count + currentOffset += count + if !page.HasMore || count == 0 { + break + } + } + return finalizeRecordExport(runtime, accumulator, startOffset, requestedLimit) +} + +func executeRecordSearchNDJSON(runtime *common.RuntimeContext, requestBody map[string]any) error { + startOffset, requestedLimit, err := recordSearchPagination(requestBody) + if err != nil { + return err + } + accumulator := &recordExportAccumulator{} + currentOffset := startOffset + remaining := requestedLimit + for remaining > 0 { + pageLimit := min(remaining, maxInlineRecordReadLimit) + body := cloneMap(requestBody) + body["offset"] = currentOffset + body["limit"] = pageLimit + data, err := baseV3Call(runtime, "POST", baseV3Path( + "bases", runtime.Str("base-token"), "tables", baseTableID(runtime), "records", "search", + ), nil, body) + if err != nil { + return err + } + page, err := parseRecordExportPage(data) + if err != nil { + return err + } + if len(page.Dataset.Records) > pageLimit { + return errs.NewInternalError( + errs.SubtypeInvalidResponse, + "record search API returned %d rows for page limit %d", len(page.Dataset.Records), pageLimit, + ) + } + if err := appendRecordExportPage(accumulator, page); err != nil { + return err + } + count := len(page.Dataset.Records) + remaining -= count + currentOffset += count + if !page.HasMore || count == 0 { + break + } + } + return finalizeRecordExport(runtime, accumulator, startOffset, requestedLimit) +} + +func executeRecordGetNDJSON(runtime *common.RuntimeContext, data map[string]any, requestedRecordCount int) error { + page, err := parseRecordExportPage(data) + if err != nil { + return err + } + page.QueryContext = cloneMap(page.QueryContext) + if page.QueryContext == nil { + page.QueryContext = make(map[string]any, 2) + } + page.QueryContext["record_scope"] = "selected_record_ids" + page.QueryContext["requested_record_count"] = requestedRecordCount + accumulator := &recordExportAccumulator{} + if err := appendRecordExportPage(accumulator, page); err != nil { + return err + } + return finalizeRecordExport(runtime, accumulator, 0, 0) +} + +func finalizeRecordExport( + runtime *common.RuntimeContext, + accumulator *recordExportAccumulator, + startOffset int, + requestedLimit int, +) error { + if !accumulator.initialized { + return errs.NewInternalError(errs.SubtypeInvalidResponse, "record export received no matrix page") + } + paths, err := resolveRecordExportPaths(runtime) + if err != nil { + return err + } + fio := runtime.FileIO() + if fio == nil { + return baseMissingFileIOError("record export requires a file I/O provider") + } + if err := ensureRecordExportTargets(fio, paths, runtime.Bool("overwrite"), runtime.Changed("output")); err != nil { + return err + } + + scanResult := output.ScanForSafety(runtime.Cmd.CommandPath(), accumulator.dataset, runtime.IO().ErrOut) + if scanResult.Blocked { + return baseContentSafetyBlockError(scanResult) + } + if scanResult.Alert != nil { + output.WriteAlertWarning(runtime.IO().ErrOut, scanResult.Alert) + } + + recordFileSizeBytes, err := saveRecordNDJSON(fio, paths.recordRelative, accumulator.dataset) + if err != nil { + return err + } + manifest := recordexport.BuildManifest(accumulator.dataset, recordexport.ManifestOptions{ + BaseToken: runtime.Str("base-token"), + TableID: baseTableID(runtime), + Rev: accumulator.rev, + QueryContext: accumulator.queryContext, + Offset: startOffset, + RequestedLimit: requestedLimit, + PageCount: accumulator.pageCount, + HasMore: accumulator.hasMore, + RecordFile: paths.recordAbsolute, + RecordFileSizeBytes: recordFileSizeBytes, + ManifestFile: paths.manifestAbsolute, + IgnoredFields: accumulator.ignoredFields, + RecordNotFound: accumulator.recordNotFound, + }) + if err := saveRecordManifest(fio, paths.manifestRelative, manifest); err != nil { + return err + } + return outputRecordExportResult(runtime, accumulator.dataset, manifest) +} + +type recordExportPaths struct { + recordRelative string + recordAbsolute string + manifestRelative string + manifestAbsolute string +} + +func resolveRecordExportPaths(runtime *common.RuntimeContext) (recordExportPaths, error) { + recordPath := strings.TrimSpace(runtime.Str("output")) + if recordPath == "" { + now := recordExportNow() + recordPath = fmt.Sprintf( + "%s_%s_%03d.ndjson", + baseTableID(runtime), now.Format("20060102_150405"), now.Nanosecond()/int(time.Millisecond), + ) + } + manifestPath := strings.TrimSuffix(recordPath, ".ndjson") + ".manifest.json" + fio := runtime.FileIO() + if fio == nil { + return recordExportPaths{}, baseMissingFileIOError("record export requires a file I/O provider") + } + recordAbsolute, err := fio.ResolvePath(recordPath) + if err != nil { + return recordExportPaths{}, baseSaveError(err) + } + manifestAbsolute, err := fio.ResolvePath(manifestPath) + if err != nil { + return recordExportPaths{}, baseSaveError(err) + } + return recordExportPaths{ + recordRelative: recordPath, recordAbsolute: recordAbsolute, + manifestRelative: manifestPath, manifestAbsolute: manifestAbsolute, + }, nil +} + +func ensureRecordExportTargets(fio fileio.FileIO, paths recordExportPaths, overwrite bool, explicitOutput bool) error { + if overwrite { + return nil + } + for _, path := range []string{paths.recordRelative, paths.manifestRelative} { + if _, err := fio.Stat(path); err == nil { + validationErr := errs.NewValidationError( + errs.SubtypeFailedPrecondition, "output file already exists: %s", path, + ).WithHint("Pass --overwrite to replace the ndjson artifact and manifest.") + if explicitOutput { + validationErr = validationErr.WithParam("--output") + } + return validationErr + } else if !errors.Is(err, fs.ErrNotExist) { + return baseSaveError(err) + } + } + return nil +} + +func saveRecordNDJSON(fio fileio.FileIO, path string, dataset recordexport.Dataset) (int64, error) { + reader, writer := io.Pipe() + writeDone := make(chan error, 1) + go func() { + err := recordexport.WriteNDJSON(writer, dataset) + _ = writer.CloseWithError(err) + writeDone <- err + }() + saveResult, saveErr := fio.Save(path, fileio.SaveOptions{ + ContentType: "application/x-ndjson", ContentLength: -1, + }, reader) + _ = reader.CloseWithError(saveErr) + writeErr := <-writeDone + if saveErr != nil { + return 0, baseSaveError(saveErr) + } + if writeErr != nil { + return 0, errs.NewInternalError(errs.SubtypeInvalidResponse, "cannot encode ndjson records: %v", writeErr).WithCause(writeErr) + } + if saveResult == nil { + return 0, errs.NewInternalError(errs.SubtypeFileIO, "record export did not report the saved ndjson size") + } + return saveResult.Size(), nil +} + +func saveRecordManifest(fio fileio.FileIO, path string, manifest recordexport.Manifest) error { + var buffer bytes.Buffer + if err := recordexport.WriteManifest(&buffer, manifest); err != nil { + return errs.NewInternalError(errs.SubtypeInvalidResponse, "cannot encode record manifest: %v", err).WithCause(err) + } + if _, err := fio.Save(path, fileio.SaveOptions{ + ContentType: "application/json", ContentLength: int64(buffer.Len()), + }, &buffer); err != nil { + return baseSaveError(err) + } + return nil +} + +func outputRecordExportResult( + runtime *common.RuntimeContext, + dataset recordexport.Dataset, + manifest recordexport.Manifest, +) error { + if jqRecords := strings.TrimSpace(runtime.Str("jq-records")); jqRecords != "" { + records := make([]any, 0, len(dataset.Records)) + for _, record := range dataset.Records { + object := make(map[string]any, len(dataset.Columns)) + for columnIndex, column := range dataset.Columns { + object[column.Name] = record.Values[columnIndex] + } + records = append(records, object) + } + return output.JqFilter(runtime.IO().Out, records, jqRecords) + } + + var value any = manifest + if runtime.Bool("minimal-stdout") { + value = manifest.Minimal() + } + scanResult := output.ScanForSafety(runtime.Cmd.CommandPath(), value, runtime.IO().ErrOut) + if scanResult.Blocked { + return baseContentSafetyBlockError(scanResult) + } + if scanResult.Alert != nil { + output.WriteAlertWarning(runtime.IO().ErrOut, scanResult.Alert) + } + if runtime.JqExpr != "" { + return output.JqFilter(runtime.IO().Out, value, runtime.JqExpr) + } + if err := output.WriteJSON(runtime.IO().Out, value); err != nil { + return errs.NewInternalError(errs.SubtypeUnknown, "cannot write record manifest to stdout: %v", err).WithCause(err) + } + return nil +} + +func appendUniqueIgnoredFields(current, incoming []recordexport.IgnoredField) []recordexport.IgnoredField { + seen := make(map[string]bool, len(current)+len(incoming)) + for _, item := range current { + seen[item.ID+"\x00"+item.Name+"\x00"+item.Reason] = true + } + for _, item := range incoming { + key := item.ID + "\x00" + item.Name + "\x00" + item.Reason + if !seen[key] { + current = append(current, item) + seen[key] = true + } + } + return current +} + +func appendUniqueStrings(current, incoming []string) []string { + seen := make(map[string]bool, len(current)+len(incoming)) + for _, item := range current { + seen[item] = true + } + for _, item := range incoming { + if !seen[item] { + current = append(current, item) + seen[item] = true + } + } + return current +} diff --git a/shortcuts/base/record_export_test.go b/shortcuts/base/record_export_test.go new file mode 100644 index 0000000000..831d05c9bb --- /dev/null +++ b/shortcuts/base/record_export_test.go @@ -0,0 +1,567 @@ +// Copyright (c) 2026 Lark Technologies Pte. Ltd. +// SPDX-License-Identifier: MIT + +package base + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "github.com/larksuite/cli/errs" + "github.com/larksuite/cli/internal/httpmock" +) + +func TestRecordListNDJSONOutputInfersFormatAndNormalizesValues(t *testing.T) { + dir := t.TempDir() + withBaseWorkingDir(t, dir) + factory, stdout, registry := newExecuteFactory(t) + registry.Register(&httpmock.Stub{ + Method: "GET", + URL: "limit=2&offset=0", + Body: map[string]any{"code": 0, "data": map[string]any{ + "rev": json.Number("42"), + "timezone": "Asia/Shanghai", + "fields": []any{"Name", "Tags", "When", "Done"}, + "field_id_list": []any{"fld_name", "fld_tags", "fld_when", "fld_done"}, + "field_type_list": []any{"text", "select", "datetime", "checkbox"}, + "record_id_list": []any{"rec_1", "rec_2"}, + "data": []any{ + []any{"Alice", nil, "2026-08-04 12:30:00", true}, + []any{nil, []any{"P0"}, nil, nil}, + }, + "has_more": false, + "query_context": map[string]any{ + "record_scope": "all_records", "field_scope": "all_fields", + }, + }}, + }) + + err := runShortcut(t, BaseRecordList, []string{ + "+record-list", "--base-token", "app_x", "--table-id", "tbl_x", + "--limit", "2", "--output", "exports/customers.ndjson", + }, factory, stdout) + if err != nil { + t.Fatalf("runShortcut() error = %v", err) + } + + var manifest map[string]any + if err := json.Unmarshal(stdout.Bytes(), &manifest); err != nil { + t.Fatalf("stdout is not a bare manifest: %v\n%s", err, stdout.String()) + } + if manifest["format"] != "ndjson" || manifest["rev"] != float64(42) || manifest["records_count"] != float64(2) || manifest["has_more"] != false { + t.Fatalf("manifest = %#v", manifest) + } + columns := manifest["columns"].(map[string]any) + recordIDColumn := columns["record_id"].(map[string]any) + if recordIDColumn["physical_type"] != "string" { + t.Fatalf("record_id column = %#v", recordIDColumn) + } + if _, exists := recordIDColumn["field_id"]; exists { + t.Fatalf("record_id column unexpectedly has field_id: %#v", recordIDColumn) + } + nameStats := columns["Name"].(map[string]any)["stats"].(map[string]any) + if nameStats["null_count"] != float64(1) || nameStats["max_length"] != float64(5) { + t.Fatalf("Name stats = %#v", nameStats) + } + tagStats := columns["Tags"].(map[string]any)["stats"].(map[string]any) + if tagStats["empty_count"] != float64(1) || tagStats["max_length"] != float64(1) || tagStats["avg_length"] != 0.5 { + t.Fatalf("Tags stats = %#v", tagStats) + } + doneColumn := columns["Done"].(map[string]any) + if doneColumn["physical_type"] != "boolean" || doneColumn["stats"].(map[string]any)["true_count"] != float64(1) { + t.Fatalf("Done column = %#v", doneColumn) + } + + recordPath := filepath.Join(dir, "exports", "customers.ndjson") + recordInfo, err := os.Stat(recordPath) + if err != nil { + t.Fatal(err) + } + if manifest["record_file_size_bytes"] != float64(recordInfo.Size()) { + t.Fatalf("record_file_size_bytes = %#v, want %d", manifest["record_file_size_bytes"], recordInfo.Size()) + } + file, err := os.Open(recordPath) + if err != nil { + t.Fatal(err) + } + defer file.Close() + scanner := bufio.NewScanner(file) + var rows []map[string]any + for scanner.Scan() { + var row map[string]any + if err := json.Unmarshal(scanner.Bytes(), &row); err != nil { + t.Fatal(err) + } + rows = append(rows, row) + } + if err := scanner.Err(); err != nil { + t.Fatal(err) + } + if len(rows) != 2 { + t.Fatalf("rows = %d", len(rows)) + } + if tags, ok := rows[0]["Tags"].([]any); !ok || len(tags) != 0 { + t.Fatalf("empty Tags = %#v", rows[0]["Tags"]) + } + if got := rows[0]["When"]; got != "2026-08-04T12:30:00+08:00" { + t.Fatalf("When = %#v", got) + } + if rows[1]["Name"] != nil { + t.Fatalf("Name = %#v, want null", rows[1]["Name"]) + } + if rows[1]["Done"] != false { + t.Fatalf("Done = %#v, want false", rows[1]["Done"]) + } + manifestFileBytes, err := os.ReadFile(filepath.Join(dir, "exports", "customers.manifest.json")) + if err != nil { + t.Fatalf("read manifest file: %v", err) + } + var savedManifest map[string]any + if err := json.Unmarshal(manifestFileBytes, &savedManifest); err != nil { + t.Fatalf("decode manifest file: %v", err) + } + if savedManifest["record_file_size_bytes"] != float64(recordInfo.Size()) { + t.Fatalf("saved record_file_size_bytes = %#v, want %d", savedManifest["record_file_size_bytes"], recordInfo.Size()) + } +} + +func TestRecordListNDJSONSerializesPagesAbove200(t *testing.T) { + dir := t.TempDir() + withBaseWorkingDir(t, dir) + factory, stdout, registry := newExecuteFactory(t) + registry.Register(&httpmock.Stub{ + Method: "GET", URL: "limit=200&offset=0", + Body: map[string]any{"code": 0, "data": recordMatrixPageWithRev(0, 200, true, "fld_name", 100)}, + }) + registry.Register(&httpmock.Stub{ + Method: "GET", URL: "limit=1&offset=200", + Body: map[string]any{"code": 0, "data": recordMatrixPageWithRev(200, 1, true, "fld_name", 101)}, + }) + + err := runShortcut(t, BaseRecordList, []string{ + "+record-list", "--base-token", "app_x", "--table-id", "tbl_x", + "--limit", "201", "--offset", "0", "--output", "paged.ndjson", "--minimal-stdout", + }, factory, stdout) + if err != nil { + t.Fatalf("runShortcut() error = %v", err) + } + var minimal map[string]any + if err := json.Unmarshal(stdout.Bytes(), &minimal); err != nil { + t.Fatal(err) + } + if len(minimal) != 5 || minimal["record_file_size_bytes"].(float64) <= 0 || minimal["records_count"] != float64(201) || minimal["has_more"] != true { + t.Fatalf("minimal stdout = %#v", minimal) + } + file, err := os.Open(filepath.Join(dir, "paged.ndjson")) + if err != nil { + t.Fatal(err) + } + defer file.Close() + scanner := bufio.NewScanner(file) + lineCount := 0 + for scanner.Scan() { + lineCount++ + } + if lineCount != 201 { + t.Fatalf("ndjson line count = %d", lineCount) + } + + manifestBytes, err := os.ReadFile(filepath.Join(dir, "paged.manifest.json")) + if err != nil { + t.Fatal(err) + } + var manifest map[string]any + if err := json.Unmarshal(manifestBytes, &manifest); err != nil { + t.Fatal(err) + } + if manifest["rev"] != float64(100) || manifest["page_count"] != float64(2) || manifest["next_offset"] != float64(201) { + t.Fatalf("manifest pagination = %#v", manifest) + } +} + +func TestRecordListNDJSONRejectsSchemaChangeWithoutPublishingFiles(t *testing.T) { + dir := t.TempDir() + withBaseWorkingDir(t, dir) + factory, stdout, registry := newExecuteFactory(t) + registry.Register(&httpmock.Stub{ + Method: "GET", URL: "limit=200&offset=0", + Body: map[string]any{"code": 0, "data": recordMatrixPage(0, 1, true, "fld_name")}, + }) + registry.Register(&httpmock.Stub{ + Method: "GET", URL: "limit=200&offset=1", + Body: map[string]any{"code": 0, "data": recordMatrixPage(1, 1, false, "fld_changed")}, + }) + + err := runShortcut(t, BaseRecordList, []string{ + "+record-list", "--base-token", "app_x", "--table-id", "tbl_x", + "--limit", "201", "--output", "changed.ndjson", + }, factory, stdout) + if err == nil { + t.Fatal("runShortcut() error = nil") + } + problem, ok := errs.ProblemOf(err) + if !ok || problem.Subtype != errs.SubtypeFailedPrecondition { + t.Fatalf("problem = %#v, err = %v", problem, err) + } + if _, statErr := os.Stat(filepath.Join(dir, "changed.ndjson")); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("changed.ndjson should not exist, stat err = %v", statErr) + } +} + +func TestRecordListNDJSONOutputCollisionHintsOverwrite(t *testing.T) { + dir := t.TempDir() + withBaseWorkingDir(t, dir) + if err := os.WriteFile(filepath.Join(dir, "existing.ndjson"), []byte("old\n"), 0o600); err != nil { + t.Fatal(err) + } + factory, stdout, registry := newExecuteFactory(t) + registry.Register(&httpmock.Stub{ + Method: "GET", URL: "limit=1&offset=0", + Body: map[string]any{"code": 0, "data": recordMatrixPage(0, 1, false, "fld_name")}, + }) + err := runShortcut(t, BaseRecordList, []string{ + "+record-list", "--base-token", "app_x", "--table-id", "tbl_x", + "--limit", "1", "--output", "existing.ndjson", + }, factory, stdout) + if err == nil { + t.Fatal("runShortcut() error = nil") + } + problem, ok := errs.ProblemOf(err) + if !ok || problem.Hint == "" || !strings.Contains(problem.Hint, "--overwrite") { + t.Fatalf("problem = %#v", problem) + } +} + +func TestRecordListNDJSONOverwriteReplacesArtifactPair(t *testing.T) { + dir := t.TempDir() + withBaseWorkingDir(t, dir) + if err := os.WriteFile(filepath.Join(dir, "existing.ndjson"), []byte("old records\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "existing.manifest.json"), []byte("old manifest\n"), 0o600); err != nil { + t.Fatal(err) + } + factory, stdout, registry := newExecuteFactory(t) + registry.Register(&httpmock.Stub{ + Method: "GET", URL: "limit=1&offset=0", + Body: map[string]any{"code": 0, "data": recordMatrixPage(0, 1, false, "fld_name")}, + }) + err := runShortcut(t, BaseRecordList, []string{ + "+record-list", "--base-token", "app_x", "--table-id", "tbl_x", + "--limit", "1", "--output", "existing.ndjson", "--overwrite", + }, factory, stdout) + if err != nil { + t.Fatalf("runShortcut() error = %v", err) + } + records, err := os.ReadFile(filepath.Join(dir, "existing.ndjson")) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(records), "old records") || !strings.Contains(string(records), `"record_id":"rec_0000"`) { + t.Fatalf("existing.ndjson = %s", records) + } + manifest, err := os.ReadFile(filepath.Join(dir, "existing.manifest.json")) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(manifest), "old manifest") || !strings.Contains(string(manifest), `"manifest_version": "v1"`) { + t.Fatalf("existing.manifest.json = %s", manifest) + } +} + +func TestRecordListNDJSONAutoNamesArtifactPair(t *testing.T) { + dir := t.TempDir() + withBaseWorkingDir(t, dir) + originalNow := recordExportNow + recordExportNow = func() time.Time { + return time.Date(2026, time.August, 4, 12, 34, 56, 789_000_000, time.UTC) + } + t.Cleanup(func() { recordExportNow = originalNow }) + + factory, stdout, registry := newExecuteFactory(t) + registry.Register(&httpmock.Stub{ + Method: "GET", URL: "limit=1&offset=0", + Body: map[string]any{"code": 0, "data": recordMatrixPage(0, 1, false, "fld_name")}, + }) + err := runShortcut(t, BaseRecordList, []string{ + "+record-list", "--base-token", "app_x", "--table-id", "tbl_x", + "--limit", "1", "--format", "ndjson", "--minimal-stdout", + }, factory, stdout) + if err != nil { + t.Fatalf("runShortcut() error = %v", err) + } + for _, name := range []string{ + "tbl_x_20260804_123456_789.ndjson", + "tbl_x_20260804_123456_789.manifest.json", + } { + if _, err := os.Stat(filepath.Join(dir, name)); err != nil { + t.Fatalf("auto output %s missing: %v", name, err) + } + } +} + +func TestRecordListNDJSONJQFiltersStdoutManifest(t *testing.T) { + dir := t.TempDir() + withBaseWorkingDir(t, dir) + factory, stdout, registry := newExecuteFactory(t) + registry.Register(&httpmock.Stub{ + Method: "GET", URL: "limit=1&offset=0", + Body: map[string]any{"code": 0, "data": recordMatrixPage(0, 1, false, "fld_name")}, + }) + err := runShortcut(t, BaseRecordList, []string{ + "+record-list", "--base-token", "app_x", "--table-id", "tbl_x", + "--limit", "1", "--output", "jq.ndjson", "--jq", ".record_file", + }, factory, stdout) + if err != nil { + t.Fatalf("runShortcut() error = %v", err) + } + canonicalDir, err := filepath.EvalSymlinks(dir) + if err != nil { + t.Fatal(err) + } + want := filepath.Join(canonicalDir, "jq.ndjson") + if got := strings.TrimSpace(stdout.String()); got != want { + t.Fatalf("stdout = %q, want %q", got, want) + } +} + +func TestRecordListNDJSONJQRecordsQueriesExportWithoutChangingArtifacts(t *testing.T) { + dir := t.TempDir() + withBaseWorkingDir(t, dir) + factory, stdout, registry := newExecuteFactory(t) + registry.Register(&httpmock.Stub{ + Method: "GET", URL: "limit=2&offset=0", + Body: map[string]any{"code": 0, "data": recordMatrixPage(0, 2, false, "fld_name")}, + }) + err := runShortcut(t, BaseRecordList, []string{ + "+record-list", "--base-token", "app_x", "--table-id", "tbl_x", + "--limit", "2", "--output", "query.ndjson", + "--jq-records", `map(select(.Name == "Name 1")) | {records_count: length, record_ids: map(.record_id)}`, + }, factory, stdout) + if err != nil { + t.Fatalf("runShortcut() error = %v", err) + } + var result map[string]any + if err := json.Unmarshal(stdout.Bytes(), &result); err != nil { + t.Fatal(err) + } + if result["records_count"] != float64(1) || !reflect.DeepEqual(result["record_ids"], []any{"rec_0001"}) { + t.Fatalf("jq result = %#v", result) + } + + records, err := os.ReadFile(filepath.Join(dir, "query.ndjson")) + if err != nil { + t.Fatal(err) + } + if strings.Count(string(records), "\n") != 2 || !strings.Contains(string(records), `"record_id":"rec_0000"`) { + t.Fatalf("query.ndjson = %s", records) + } + manifest, err := os.ReadFile(filepath.Join(dir, "query.manifest.json")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(manifest), `"records_count": 2`) { + t.Fatalf("query.manifest.json = %s", manifest) + } +} + +func TestRecordListJQRecordsValidatesOutputContractBeforeRequest(t *testing.T) { + tests := []struct { + name string + args []string + want string + }{ + { + name: "requires ndjson", + args: []string{"--format", "json", "--jq-records", "length"}, + want: "--jq-records requires --format ndjson", + }, + { + name: "conflicts with manifest jq", + args: []string{"--format", "ndjson", "--jq", ".record_file", "--jq-records", "length"}, + want: "--jq-records and --jq are mutually exclusive", + }, + { + name: "conflicts with minimal stdout", + args: []string{"--format", "ndjson", "--minimal-stdout", "--jq-records", "length"}, + want: "--jq-records and --minimal-stdout are mutually exclusive", + }, + { + name: "validates expression", + args: []string{"--format", "ndjson", "--jq-records", "invalid["}, + want: "invalid jq expression", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + factory, stdout, _ := newExecuteFactory(t) + args := append([]string{ + "+record-list", "--base-token", "app_x", "--table-id", "tbl_x", "--limit", "1", + }, tt.args...) + err := runShortcut(t, BaseRecordList, args, factory, stdout) + if err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("runShortcut() error = %v, want containing %q", err, tt.want) + } + }) + } +} + +func TestRecordSearchNDJSONPaginatesAndPreservesSearchBody(t *testing.T) { + dir := t.TempDir() + withBaseWorkingDir(t, dir) + factory, stdout, registry := newExecuteFactory(t) + first := &httpmock.Stub{ + Method: "POST", + URL: "/open-apis/base/v3/bases/app_x/tables/tbl_x/records/search", + BodyFilter: func(body []byte) bool { + return strings.Contains(string(body), `"offset":0`) && strings.Contains(string(body), `"limit":200`) + }, + Body: map[string]any{"code": 0, "data": recordMatrixPage(0, 200, true, "fld_name")}, + } + second := &httpmock.Stub{ + Method: "POST", + URL: "/open-apis/base/v3/bases/app_x/tables/tbl_x/records/search", + BodyFilter: func(body []byte) bool { + return strings.Contains(string(body), `"offset":200`) && strings.Contains(string(body), `"limit":1`) + }, + Body: map[string]any{"code": 0, "data": recordMatrixPage(200, 1, false, "fld_name")}, + } + registry.Register(first) + registry.Register(second) + + err := runShortcut(t, BaseRecordSearch, []string{ + "+record-search", "--base-token", "app_x", "--table-id", "tbl_x", + "--keyword", "Name", "--search-field", "Name", + "--limit", "201", "--output", "search.ndjson", "--filter-json", `{"logic":"and","conditions":[]}`, + }, factory, stdout) + if err != nil { + t.Fatalf("runShortcut() error = %v", err) + } + for _, body := range [][]byte{first.CapturedBody, second.CapturedBody} { + if !strings.Contains(string(body), `"keyword":"Name"`) || !strings.Contains(string(body), `"filter":{"conditions":[],"logic":"and"}`) { + t.Fatalf("search body lost query fields: %s", body) + } + } + manifestBytes, err := os.ReadFile(filepath.Join(dir, "search.manifest.json")) + if err != nil { + t.Fatal(err) + } + var manifest map[string]any + if err := json.Unmarshal(manifestBytes, &manifest); err != nil { + t.Fatal(err) + } + if manifest["records_count"] != float64(201) || manifest["page_count"] != float64(2) { + t.Fatalf("manifest = %#v", manifest) + } +} + +func TestRecordGetNDJSONWritesArtifact(t *testing.T) { + dir := t.TempDir() + withBaseWorkingDir(t, dir) + factory, stdout, registry := newExecuteFactory(t) + registry.Register(&httpmock.Stub{ + Method: "POST", + URL: "/open-apis/base/v3/bases/app_x/tables/tbl_x/records/batch_get", + Body: map[string]any{"code": 0, "data": map[string]any{ + "timezone": "UTC", + "fields": []any{"Name"}, + "field_id_list": []any{"fld_name"}, + "field_type_list": []any{"text"}, + "record_id_list": []any{"rec_1"}, + "data": []any{[]any{"Alice"}}, + "query_context": map[string]any{ + "record_scope": "all_records", "field_scope": "all_fields", + }, + }}, + }) + err := runShortcut(t, BaseRecordGet, []string{ + "+record-get", "--base-token", "app_x", "--table-id", "tbl_x", + "--record-id", "rec_1", "--output", "get.ndjson", "--minimal-stdout", + }, factory, stdout) + if err != nil { + t.Fatalf("runShortcut() error = %v", err) + } + data, err := os.ReadFile(filepath.Join(dir, "get.ndjson")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), `"record_id":"rec_1"`) || !strings.Contains(string(data), `"Name":"Alice"`) { + t.Fatalf("get.ndjson = %s", data) + } + manifestData, err := os.ReadFile(filepath.Join(dir, "get.manifest.json")) + if err != nil { + t.Fatal(err) + } + var manifest map[string]any + if err := json.Unmarshal(manifestData, &manifest); err != nil { + t.Fatal(err) + } + queryContext := manifest["query_context"].(map[string]any) + if queryContext["record_scope"] != "selected_record_ids" || + queryContext["requested_record_count"] != float64(1) || + queryContext["field_scope"] != "all_fields" { + t.Fatalf("query_context = %#v", queryContext) + } +} + +func TestRecordNDJSONFlagValidation(t *testing.T) { + factory, stdout, _ := newExecuteFactory(t) + err := runShortcut(t, BaseRecordList, []string{ + "+record-list", "--base-token", "app_x", "--table-id", "tbl_x", + "--format", "json", "--output", "out.ndjson", + }, factory, stdout) + assertInvalidArgumentValidation(t, err, "--output", []string{"--output", "--format"}, "conflicts") + + err = runShortcut(t, BaseRecordList, []string{ + "+record-list", "--base-token", "app_x", "--table-id", "tbl_x", + "--limit", "2001", "--output", "out.ndjson", + }, factory, stdout) + assertInvalidArgumentValidation(t, err, "--limit", nil, "between 1 and 2000") + + err = runShortcut(t, BaseRecordSearch, []string{ + "+record-search", "--base-token", "app_x", "--table-id", "tbl_x", + "--json", `{"keyword":"Alice","search_fields":["Name"],"limit":2001}`, + "--output", "out.ndjson", + }, factory, stdout) + assertInvalidArgumentValidation(t, err, "--json", nil, "between 1 and 2000") + + err = runShortcut(t, BaseRecordSearch, []string{ + "+record-search", "--base-token", "app_x", "--table-id", "tbl_x", + "--json", `{"keyword":"Alice","search_fields":["Name"],"limit":201}`, + "--format", "json", + }, factory, stdout) + assertInvalidArgumentValidation(t, err, "--json", nil, "between 1 and 200") +} + +func recordMatrixPage(start, count int, hasMore bool, fieldID string) map[string]any { + recordIDs := make([]any, 0, count) + rows := make([]any, 0, count) + for index := 0; index < count; index++ { + number := start + index + recordIDs = append(recordIDs, fmt.Sprintf("rec_%04d", number)) + rows = append(rows, []any{fmt.Sprintf("Name %d", number)}) + } + return map[string]any{ + "timezone": "UTC", + "fields": []any{"Name"}, + "field_id_list": []any{fieldID}, + "field_type_list": []any{"text"}, + "record_id_list": recordIDs, + "data": rows, + "has_more": hasMore, + } +} + +func recordMatrixPageWithRev(start, count int, hasMore bool, fieldID string, rev int64) map[string]any { + page := recordMatrixPage(start, count, hasMore, fieldID) + page["rev"] = json.Number(fmt.Sprintf("%d", rev)) + return page +} diff --git a/shortcuts/base/record_get.go b/shortcuts/base/record_get.go index 6ba1eda592..d148da5d8c 100644 --- a/shortcuts/base/record_get.go +++ b/shortcuts/base/record_get.go @@ -26,20 +26,29 @@ var BaseRecordGet = common.Shortcut{ recordProjectionAliasFlag("field-names"), {Name: "json", Desc: `JSON object with record_id_list, e.g. {"record_id_list":["rec_xxx"]}`}, recordReadFormatFlag(), + recordOutputFlag(), + recordMinimalStdoutFlag(), + recordJQRecordsFlag(), + recordOverwriteFlag(), }, + Normalize: normalizeRecordReadOutput, + JQFormats: []string{"ndjson"}, Validate: func(ctx context.Context, runtime *common.RuntimeContext) error { if err := validateRecordReadFormat(runtime); err != nil { return err } + if err := validateRecordExportFlags(runtime); err != nil { + return err + } return validateRecordSelection(runtime) }, Tips: []string{ "Example: lark-cli base +record-get --base-token --table-id --record-id ", "Example with projection: lark-cli base +record-get --base-token --table-id --record-id rec_001 --record-id rec_002 --field-id Name --field-id Status", - "Default output is markdown; pass --format json to get the raw JSON envelope.", + "Example for analysis input: lark-cli base +record-get --base-token --table-id --record-id --field-id --output ./record.ndjson --minimal-stdout", + recordAnalysisOutputTip, "Use --field-id as a projection boundary to avoid loading large cell values into context when they are not needed.", "Use +record-get when record_id is already known; otherwise use +record-search or +record-list.", - "Agent hint: follow the lark-base record read SOP for record read routing.", }, DryRun: dryRunRecordGet, PostMount: func(cmd *cobra.Command) { diff --git a/shortcuts/base/record_json_shorthand_test.go b/shortcuts/base/record_json_shorthand_test.go index 25aeb4a490..c9157810f8 100644 --- a/shortcuts/base/record_json_shorthand_test.go +++ b/shortcuts/base/record_json_shorthand_test.go @@ -63,11 +63,11 @@ func TestRecordSearchGetKeepRequestBodyJSON(t *testing.T) { } } -// Enum 已接入：help 描述携带枚举后缀（框架对带 Enum 的 flag 自动追加 " (markdown|json)"） +// Enum 已接入：help 描述携带枚举后缀。 func TestRecordReadFormatFlagCarriesEnum(t *testing.T) { cmd := mountBaseShortcutFlags(t, BaseRecordList, "+record-list") usage := cmd.Flags().Lookup("format").Usage - if !strings.Contains(usage, "(markdown|json)") { + if !strings.Contains(usage, "(markdown|json|ndjson)") { t.Fatalf("format usage missing enum suffix: %q", usage) } } diff --git a/shortcuts/base/record_list.go b/shortcuts/base/record_list.go index d65636040a..7ffda987b0 100644 --- a/shortcuts/base/record_list.go +++ b/shortcuts/base/record_list.go @@ -27,12 +27,17 @@ var BaseRecordList = common.Shortcut{ recordFilterFlag(), recordSortFlag(), {Name: "offset", Type: "int", Default: "0", Desc: "pagination offset"}, - {Name: "limit", Aliases: []string{"page-size"}, Type: "int", Default: "100", Desc: "pagination size, range 1-200"}, + {Name: "limit", Aliases: []string{"page-size"}, Type: "int", Default: "100", Desc: "maximum records to return; range 1-200, or 1-2000 for ndjson"}, recordReadFormatFlag(), + recordOutputFlag(), + recordMinimalStdoutFlag(), + recordJQRecordsFlag(), + recordOverwriteFlag(), }, Tips: []string{ "Example: lark-cli base +record-list --base-token --table-id --limit 50", "Example with projection: lark-cli base +record-list --base-token --table-id --field-id Name --field-id Status --limit 50", + "Example for analysis: lark-cli base +record-list --base-token --table-id --field-id Name --field-id Status --limit 2000 --output ./records.ndjson --minimal-stdout", `Text equality filter: --filter-json '{"logic":"and","conditions":[["Title","==","Launch plan"]]}'`, `Text contains/like filter: --filter-json '{"logic":"and","conditions":[["Title","intersects","urgent"]]}'`, `Number equality filter: --filter-json '{"logic":"and","conditions":[["Score","==",95]]}'`, @@ -40,14 +45,19 @@ var BaseRecordList = common.Shortcut{ `Option intersection filter: --filter-json '{"logic":"and","conditions":[["Tags","intersects",["P0","Blocked"]]]}'`, `Sort priority follows --sort-json array order: --sort-json '[{"field":"Updated","desc":true},{"field":"Title","desc":false}]'`, formatRecordQueryPriorityTip(), - "Default output is markdown; pass --format json to get the raw JSON envelope.", + recordAnalysisOutputTip, "Use --field-id repeatedly to keep output small and aligned with the task.", }, + Normalize: normalizeRecordReadOutput, + JQFormats: []string{"ndjson"}, Validate: func(ctx context.Context, runtime *common.RuntimeContext) error { if err := validateRecordReadFormat(runtime); err != nil { return err } - if _, err := common.ValidatePageSizeTyped(runtime, "limit", 100, 1, 200); err != nil { + if err := validateRecordExportFlags(runtime); err != nil { + return err + } + if err := validateRecordReadLimit(runtime, 100); err != nil { return err } if _, err := recordProjectionFields(runtime); err != nil { @@ -74,7 +84,7 @@ func recordReadFormatFlag() common.Flag { return common.Flag{ Name: "format", Default: "markdown", - Enum: []string{"markdown", "json"}, - Desc: "output format: markdown (default) | json", + Enum: []string{"markdown", "json", "ndjson"}, + Desc: "output format: markdown (default display) | json raw matrix | ndjson typed artifact (preferred for analysis)", } } diff --git a/shortcuts/base/record_markdown.go b/shortcuts/base/record_markdown.go index ea5c86b9da..7123985f04 100644 --- a/shortcuts/base/record_markdown.go +++ b/shortcuts/base/record_markdown.go @@ -17,10 +17,10 @@ const maxRecordMarkdownIgnoredFields = 20 func validateRecordReadFormat(runtime *common.RuntimeContext) error { switch runtime.Str("format") { - case "", "json", "markdown": + case "", "json", "markdown", "ndjson": return nil default: - return baseValidationErrorf("--format must be json or markdown") + return baseValidationErrorf("--format must be json, markdown, or ndjson") } } diff --git a/shortcuts/base/record_markdown_test.go b/shortcuts/base/record_markdown_test.go index 09775b2f14..b1df5c3479 100644 --- a/shortcuts/base/record_markdown_test.go +++ b/shortcuts/base/record_markdown_test.go @@ -156,7 +156,11 @@ func TestRenderRecordMarkdownIncludesMissingRecords(t *testing.T) { func TestRenderRecordMarkdownTruncatesIgnoredFields(t *testing.T) { ignored := make([]interface{}, maxRecordMarkdownIgnoredFields+2) for i := range ignored { - ignored[i] = fmt.Sprintf("Field%d", i+1) + ignored[i] = map[string]interface{}{ + "id": fmt.Sprintf("fld_%d", i+1), + "name": fmt.Sprintf("Field%d", i+1), + "reason": "UNSUPPORTED", + } } got, err := renderRecordMarkdown(map[string]interface{}{ "fields": []interface{}{"Name"}, diff --git a/shortcuts/base/record_ops.go b/shortcuts/base/record_ops.go index 8e882adf6f..b1eb3e84ee 100644 --- a/shortcuts/base/record_ops.go +++ b/shortcuts/base/record_ops.go @@ -19,8 +19,9 @@ const maxBatchGetSelectFieldCount = 100 const maxRecordSearchSelectFieldCount = 50 var recordCellValueHappyPathTips = []string{ - `CellValue happy path: text/phone/url -> "text"; number/currency/percent/rating -> 12.5; select (multiple=false) -> "Todo"; select (multiple=true) -> ["Tag A","Tag B"]; datetime -> "2026-03-24 10:00:00"; checkbox -> true/false.`, + `CellValue happy path: text/phone/url -> "text"; number/currency/percent/rating -> 12.5; select -> ["Todo"] or ["Tag A","Tag B"] (when multiple=false, the array can contain only one option); datetime -> "2026-03-24 10:00"; checkbox -> true/false.`, `ID-based CellValue: user/group/link fields use arrays like [{"id":"ou_xxx"}], [{"id":"oc_xxx"}], [{"id":"rec_xxx"}]; location uses {"lng":116.397428,"lat":39.90923}; null clears a cell when allowed.`, + "User and group fields always use arrays; when multiple=false, the array can contain only one item.", "Do not guess user/chat/linked-record IDs or location coordinates; resolve them first with the relevant contact/im/record lookup flow.", "Use lark-base-cell-value.md for complex CellValue shapes and special field types; do not invent values for fields not covered by the happy path.", } @@ -226,9 +227,13 @@ func dryRunRecordList(_ context.Context, runtime *common.RuntimeContext) *common offset = 0 } limit := runtime.Int("limit") + requestLimit := limit + if runtime.Str("format") == "ndjson" { + requestLimit = min(limit, maxInlineRecordReadLimit) + } params := url.Values{} params.Set("offset", strconv.Itoa(offset)) - params.Set("limit", strconv.Itoa(limit)) + params.Set("limit", strconv.Itoa(requestLimit)) fields, err := recordProjectionFields(runtime) if err != nil { return common.NewDryRunAPI() @@ -243,10 +248,17 @@ func dryRunRecordList(_ context.Context, runtime *common.RuntimeContext) *common return common.NewDryRunAPI() } path := "/open-apis/base/v3/bases/:base_token/tables/:table_id/records?" + params.Encode() - return common.NewDryRunAPI(). + dry := common.NewDryRunAPI(). GET(path). Set("base_token", runtime.Str("base-token")). Set("table_id", baseTableID(runtime)) + if runtime.Str("format") == "ndjson" { + dry.Set("export_format", "ndjson").Set("requested_limit", limit) + if outputPath := strings.TrimSpace(runtime.Str("output")); outputPath != "" { + dry.Set("output", outputPath) + } + } + return dry } func dryRunRecordGet(_ context.Context, runtime *common.RuntimeContext) *common.DryRunAPI { @@ -254,11 +266,18 @@ func dryRunRecordGet(_ context.Context, runtime *common.RuntimeContext) *common. if err != nil { return common.NewDryRunAPI() } - return common.NewDryRunAPI(). + dry := common.NewDryRunAPI(). POST("/open-apis/base/v3/bases/:base_token/tables/:table_id/records/batch_get"). Body(recordGetBatchBody(selection)). Set("base_token", runtime.Str("base-token")). Set("table_id", baseTableID(runtime)) + if runtime.Str("format") == "ndjson" { + dry.Set("export_format", "ndjson") + if outputPath := strings.TrimSpace(runtime.Str("output")); outputPath != "" { + dry.Set("output", outputPath) + } + } + return dry } func dryRunRecordSearch(_ context.Context, runtime *common.RuntimeContext) *common.DryRunAPI { @@ -268,7 +287,18 @@ func dryRunRecordSearch(_ context.Context, runtime *common.RuntimeContext) *comm } else { body, _ = recordSearchFlagBody(runtime) } - return common.NewDryRunAPI(). + dry := common.NewDryRunAPI() + if runtime.Str("format") == "ndjson" && body != nil { + _, requestedLimit, err := recordSearchPagination(body) + if err == nil { + body["limit"] = min(requestedLimit, maxInlineRecordReadLimit) + dry.Set("export_format", "ndjson").Set("requested_limit", requestedLimit) + if outputPath := strings.TrimSpace(runtime.Str("output")); outputPath != "" { + dry.Set("output", outputPath) + } + } + } + return dry. POST("/open-apis/base/v3/bases/:base_token/tables/:table_id/records/search"). Body(body). Set("base_token", runtime.Str("base-token")). @@ -523,7 +553,7 @@ func executeRecordList(runtime *common.RuntimeContext) error { offset = 0 } limit := runtime.Int("limit") - params := map[string]interface{}{"offset": offset, "limit": limit} + params := map[string]interface{}{} fields, err := recordProjectionFields(runtime) if err != nil { return err @@ -537,6 +567,11 @@ func executeRecordList(runtime *common.RuntimeContext) error { if err := applyRecordQueryToParams(runtime, params); err != nil { return err } + if runtime.Str("format") == "ndjson" { + return executeRecordListNDJSON(runtime, params, offset, limit) + } + params["offset"] = offset + params["limit"] = limit data, err := baseV3Call(runtime, "GET", baseV3Path("bases", runtime.Str("base-token"), "tables", baseTableID(runtime), "records"), params, nil) if err != nil { return err @@ -564,6 +599,9 @@ func executeRecordGet(runtime *common.RuntimeContext) error { if runtime.Str("format") == "markdown" { return outputRecordGetMarkdown(runtime, data) } + if runtime.Str("format") == "ndjson" { + return executeRecordGetNDJSON(runtime, data, len(selection.recordIDs)) + } runtime.Out(data, nil) return nil } @@ -579,6 +617,9 @@ func executeRecordSearch(runtime *common.RuntimeContext) error { if err != nil { return err } + if runtime.Str("format") == "ndjson" { + return executeRecordSearchNDJSON(runtime, body) + } data, err := baseV3Call(runtime, "POST", baseV3Path("bases", runtime.Str("base-token"), "tables", baseTableID(runtime), "records", "search"), nil, body) if err != nil { return err diff --git a/shortcuts/base/record_query.go b/shortcuts/base/record_query.go index bf4687a47b..6218c37227 100644 --- a/shortcuts/base/record_query.go +++ b/shortcuts/base/record_query.go @@ -6,6 +6,7 @@ package base import ( "encoding/json" "fmt" + "math" "net/url" "strings" @@ -234,6 +235,9 @@ func validateRecordSearchFlags(runtime *common.RuntimeContext) error { if err := validateRecordReadFormat(runtime); err != nil { return err } + if err := validateRecordExportFlags(runtime); err != nil { + return err + } jsonRaw := strings.TrimSpace(runtime.Str("json")) if jsonRaw != "" { if exclusiveParams := recordSearchJSONExclusiveFlagParams(runtime); len(exclusiveParams) > 0 { @@ -251,8 +255,22 @@ func validateRecordSearchFlags(runtime *common.RuntimeContext) error { WithParams(invalidParams...). WithHint("Put keyword, search, projection, view, and pagination fields inside --json, or omit --json.") } - _, err := recordSearchJSONBody(runtime) - return err + body, err := recordSearchJSONBody(runtime) + if err != nil { + return err + } + _, limit, err := recordSearchPagination(body) + if err != nil { + return withValidationParam(err, "--json") + } + maximum := maxInlineRecordReadLimit + if runtime.Str("format") == "ndjson" { + maximum = maxNDJSONRecordReadLimit + } + if limit > maximum { + return withValidationParam(baseFlagErrorf("limit must be between 1 and %d; got %d", maximum, limit), "--json") + } + return nil } if strings.TrimSpace(runtime.Str("keyword")) == "" { return baseFlagErrorf("--keyword is required unless --json is used") @@ -260,7 +278,7 @@ func validateRecordSearchFlags(runtime *common.RuntimeContext) error { if len(runtime.StrArray("search-field")) == 0 { return baseFlagErrorf("--search-field is required unless --json is used") } - if _, err := common.ValidatePageSizeTyped(runtime, "limit", 10, 1, 200); err != nil { + if err := validateRecordReadLimit(runtime, 10); err != nil { return err } if _, err := recordSearchProjectionFields(runtime); err != nil { @@ -269,6 +287,54 @@ func validateRecordSearchFlags(runtime *common.RuntimeContext) error { return validateRecordQueryOptions(runtime) } +func recordSearchPagination(body map[string]any) (int, int, error) { + offset := 0 + if raw, exists := body["offset"]; exists { + value, err := recordSearchInteger(raw, "offset") + if err != nil { + return 0, 0, err + } + if value < 0 { + return 0, 0, baseFlagErrorf("offset must be greater than or equal to 0; got %d", value) + } + offset = value + } + limit := 10 + if raw, exists := body["limit"]; exists { + value, err := recordSearchInteger(raw, "limit") + if err != nil { + return 0, 0, err + } + if value < 1 { + return 0, 0, baseFlagErrorf("limit must be greater than or equal to 1; got %d", value) + } + limit = value + } + return offset, limit, nil +} + +func recordSearchInteger(raw any, name string) (int, error) { + switch value := raw.(type) { + case int: + return value, nil + case int64: + return int(value), nil + case json.Number: + parsed, err := value.Int64() + if err != nil { + return 0, baseFlagErrorf("%s must be an integer; got %v", name, raw) + } + return int(parsed), nil + case float64: + if math.Trunc(value) != value { + return 0, baseFlagErrorf("%s must be an integer; got %v", name, raw) + } + return int(value), nil + default: + return 0, baseFlagErrorf("%s must be an integer; got %T", name, raw) + } +} + func recordSearchJSONExclusiveFlagParams(runtime *common.RuntimeContext) []string { names := []string{ "keyword", diff --git a/shortcuts/base/record_search.go b/shortcuts/base/record_search.go index 600bc71ff0..35400f021c 100644 --- a/shortcuts/base/record_search.go +++ b/shortcuts/base/record_search.go @@ -30,15 +30,20 @@ var BaseRecordSearch = common.Shortcut{ recordFilterFlag(), recordSortFlag(), {Name: "offset", Type: "int", Default: "0", Desc: "pagination offset"}, - {Name: "limit", Aliases: []string{"page-size"}, Type: "int", Default: "10", Desc: "pagination size, range 1-200"}, + {Name: "limit", Aliases: []string{"page-size"}, Type: "int", Default: "10", Desc: "maximum records to return; range 1-200, or 1-2000 for ndjson"}, recordReadFormatFlag(), + recordOutputFlag(), + recordMinimalStdoutFlag(), + recordJQRecordsFlag(), + recordOverwriteFlag(), }, Tips: []string{ - `Happy path fields: keyword (string), search_fields (1-20 field names/ids), select_fields (optional projection, <=50), view_id (optional), offset (default 0), limit (default 10, range 1-200).`, - "JSON constraints: keyword length >=1; search_fields length 1-20; select_fields length <=50; offset >=0 defaults to 0; limit range 1-200 defaults to 10.", + `Happy path fields: keyword (string), search_fields (1-20 field names/ids), select_fields (optional projection, <=50), view_id (optional), offset (default 0), limit (default 10; inline range 1-200, ndjson range 1-2000).`, + "JSON constraints: keyword length >=1; search_fields length 1-20; select_fields length <=50; offset >=0 defaults to 0; limit defaults to 10 and follows the active output format's range.", "view_id scopes search to records in that view; when select_fields is omitted, returned fields follow that view's visible fields.", `Example: lark-cli base +record-search --base-token --table-id --keyword Alice --search-field Name --field-id Name --field-id Status --limit 20`, `Example with filter/sort JSON: lark-cli base +record-search --base-token --table-id --keyword Alice --search-field Name --filter-json @filter.json --sort-json '[{"field":"Updated","desc":true}]'`, + `Example for analysis: lark-cli base +record-search --base-token --table-id --keyword Alice --search-field Name --field-id Name --field-id Status --limit 2000 --output ./records.ndjson --minimal-stdout`, `Text equality filter: --filter-json '{"logic":"and","conditions":[["Title","==","Launch plan"]]}'`, `Text contains/like filter: --filter-json '{"logic":"and","conditions":[["Title","intersects","urgent"]]}'`, `Option intersection filter: --filter-json '{"logic":"and","conditions":[["Tags","intersects",["P0","Blocked"]]]}'`, @@ -46,8 +51,10 @@ var BaseRecordSearch = common.Shortcut{ formatRecordQueryPriorityTip(), "Use +record-search for keyword matching; use --filter-json for structured conditions and --sort-json for result ordering.", "Use --json only when you need to pass the full search body directly.", - "Default output is markdown; pass --format json to get the raw JSON envelope.", + recordAnalysisOutputTip, }, + Normalize: normalizeRecordReadOutput, + JQFormats: []string{"ndjson"}, Validate: func(ctx context.Context, runtime *common.RuntimeContext) error { return validateRecordSearchFlags(runtime) }, diff --git a/shortcuts/base/recordexport/dataset.go b/shortcuts/base/recordexport/dataset.go new file mode 100644 index 0000000000..790b7d4b23 --- /dev/null +++ b/shortcuts/base/recordexport/dataset.go @@ -0,0 +1,416 @@ +// Copyright (c) 2026 Lark Technologies Pte. Ltd. +// SPDX-License-Identifier: MIT + +// Package recordexport converts the Base OpenAPI record matrix into a stable, +// typed row model that output formats can share. The matrix is intentionally +// parsed only once at the package boundary; exporters never depend on loose +// map keys or parallel arrays. +package recordexport + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "time" +) + +const RecordIDColumnName = "record_id" + +// ValueKind is the outer decoded JSON representation used for lightweight +// shape validation. Nested objects remain map[string]any values. +type ValueKind string + +const ( + KindString ValueKind = "string" + KindNumber ValueKind = "number" + KindBoolean ValueKind = "boolean" + KindObject ValueKind = "object" +) + +type fieldTypeSpec struct { + kind ValueKind + repeated bool + physicalType string +} + +// specForFieldType is the single mapping from the OpenAPI field_type contract +// to NDJSON outer-shape validation and manifest physical types. Record values +// remain decoded JSON values; the CLI does not construct Go structs for cells. +func specForFieldType(fieldType string) (fieldTypeSpec, bool) { + switch fieldType { + case "text", "formula", "lookup", "auto_number", "datetime", "created_at", "updated_at", "not_support": + return fieldTypeSpec{kind: KindString, physicalType: "string|null"}, true + case "number": + return fieldTypeSpec{kind: KindNumber, physicalType: "number|null"}, true + case "checkbox": + return fieldTypeSpec{kind: KindBoolean, physicalType: "boolean"}, true + case "location": + return fieldTypeSpec{ + kind: KindObject, physicalType: "struct|null", + }, true + case "select": + return fieldTypeSpec{kind: KindString, repeated: true, physicalType: "array"}, true + case "user", "group_chat", "created_by", "updated_by": + return fieldTypeSpec{ + kind: KindObject, repeated: true, + physicalType: "array>", + }, true + case "link": + return fieldTypeSpec{ + kind: KindObject, repeated: true, + physicalType: "array>", + }, true + case "attachment": + return fieldTypeSpec{ + kind: KindObject, repeated: true, + physicalType: "array>", + }, true + default: + return fieldTypeSpec{}, false + } +} + +// Column is the format-neutral schema used by all record exporters. FieldID +// and FieldType are empty only for the synthetic record_id system column. +type Column struct { + Name string + FieldID string + FieldType string + System bool +} + +// PhysicalType renders the compact, engine-neutral type used in manifests. +func (c Column) PhysicalType() string { + if c.System { + return "string" + } + spec, ok := specForFieldType(c.FieldType) + if !ok { + return "" + } + return spec.physicalType +} + +// Record stores decoded JSON values in the same order as Dataset.Columns. +// Object cells remain map[string]any and are not converted into Go structs. +type Record struct { + Values []any +} + +// Dataset is the stable tabular model consumed by NDJSON now and by future +// JSON-array or Parquet exporters. SourceColumns retains the complete OpenAPI +// schema for cross-page consistency checks, while Columns contains the actual +// exported columns (including the synthetic record_id column). +type Dataset struct { + Timezone string + SourceColumns []Column + Columns []Column + Records []Record +} + +// IgnoredField mirrors the structured OpenAPI read warning. +type IgnoredField struct { + ID string `json:"id"` + Name string `json:"name"` + Reason string `json:"reason"` +} + +// Page contains one parsed matrix page and its query metadata. +type Page struct { + Dataset Dataset + Rev *int64 + HasMore bool + IgnoredFields []IgnoredField + QueryContext map[string]any + RecordNotFound []string +} + +// MatrixError means the OpenAPI response does not satisfy its parallel-array +// contract. The command boundary wraps it as a typed invalid-response error. +type MatrixError struct { + Reason string +} + +func (e *MatrixError) Error() string { return "invalid record matrix: " + e.Reason } + +// SchemaChangedError protects a multi-page export from mixing schemas. +type SchemaChangedError struct { + Reason string +} + +func (e *SchemaChangedError) Error() string { + return "record schema changed between pages: " + e.Reason +} + +// ParseMatrix converts the current OpenAPI matrix shape into a typed page. +func ParseMatrix(data map[string]any) (Page, error) { + timezone, ok := data["timezone"].(string) + if !ok || strings.TrimSpace(timezone) == "" { + return Page{}, &MatrixError{Reason: "timezone must be a non-empty string"} + } + fields, err := stringList(data["fields"], "fields") + if err != nil { + return Page{}, err + } + fieldIDs, err := stringList(data["field_id_list"], "field_id_list") + if err != nil { + return Page{}, err + } + fieldTypes, err := stringList(data["field_type_list"], "field_type_list") + if err != nil { + return Page{}, err + } + if len(fields) != len(fieldIDs) || len(fields) != len(fieldTypes) { + return Page{}, &MatrixError{Reason: fmt.Sprintf( + "fields, field_id_list, and field_type_list lengths differ (%d, %d, %d)", + len(fields), len(fieldIDs), len(fieldTypes), + )} + } + + sourceColumns := make([]Column, 0, len(fields)) + exportColumns := []Column{{ + Name: RecordIDColumnName, System: true, + }} + exportSourceIndexes := make([]int, 0, len(fields)) + for index := range fields { + column, err := sourceColumn(fields[index], fieldIDs[index], fieldTypes[index]) + if err != nil { + return Page{}, err + } + sourceColumns = append(sourceColumns, column) + // The system join key intentionally wins over a same-named Base field. + if column.Name == RecordIDColumnName { + continue + } + exportColumns = append(exportColumns, column) + exportSourceIndexes = append(exportSourceIndexes, index) + } + + recordIDs, err := stringList(data["record_id_list"], "record_id_list") + if err != nil { + return Page{}, err + } + rawRows, ok := data["data"].([]any) + if !ok { + return Page{}, &MatrixError{Reason: "data must be an array of rows"} + } + if len(recordIDs) != len(rawRows) { + return Page{}, &MatrixError{Reason: fmt.Sprintf( + "record_id_list and data lengths differ (%d, %d)", len(recordIDs), len(rawRows), + )} + } + + records := make([]Record, 0, len(rawRows)) + for rowIndex, rawRow := range rawRows { + row, ok := rawRow.([]any) + if !ok { + return Page{}, &MatrixError{Reason: fmt.Sprintf("data row %d must be an array", rowIndex+1)} + } + if len(row) != len(sourceColumns) { + return Page{}, &MatrixError{Reason: fmt.Sprintf( + "data row %d has %d cells; schema has %d columns", rowIndex+1, len(row), len(sourceColumns), + )} + } + values := make([]any, 1, len(exportColumns)) + values[0] = recordIDs[rowIndex] + for _, sourceIndex := range exportSourceIndexes { + value, err := normalizeCell(sourceColumns[sourceIndex], row[sourceIndex], timezone) + if err != nil { + return Page{}, &MatrixError{Reason: fmt.Sprintf( + "row %d column %q: %v", rowIndex+1, sourceColumns[sourceIndex].Name, err, + )} + } + values = append(values, value) + } + records = append(records, Record{Values: values}) + } + + page := Page{Dataset: Dataset{ + Timezone: timezone, SourceColumns: sourceColumns, Columns: exportColumns, Records: records, + }} + if raw, exists := data["rev"]; exists && raw != nil { + value, ok := raw.(json.Number) + if !ok { + return Page{}, &MatrixError{Reason: fmt.Sprintf("rev must be an integer, got %T", raw)} + } + rev, err := value.Int64() + if err != nil || rev < 0 { + return Page{}, &MatrixError{Reason: "rev must be a non-negative integer"} + } + page.Rev = &rev + } + if raw, exists := data["has_more"]; exists { + value, ok := raw.(bool) + if !ok { + return Page{}, &MatrixError{Reason: "has_more must be a boolean"} + } + page.HasMore = value + } + if raw, exists := data["query_context"]; exists && raw != nil { + value, ok := raw.(map[string]any) + if !ok { + return Page{}, &MatrixError{Reason: "query_context must be an object"} + } + page.QueryContext = value + } + if raw, exists := data["ignored_fields"]; exists && raw != nil { + page.IgnoredFields, err = ignoredFieldList(raw) + if err != nil { + return Page{}, err + } + } + if raw, exists := data["record_not_found"]; exists && raw != nil { + page.RecordNotFound, err = stringList(raw, "record_not_found") + if err != nil { + return Page{}, err + } + } + return page, nil +} + +// AppendPage appends rows only after proving that the complete source schema +// and timezone still match the first page. +func (d *Dataset) AppendPage(page Page) error { + if d.Timezone != page.Dataset.Timezone { + return &SchemaChangedError{Reason: fmt.Sprintf("timezone changed from %q to %q", d.Timezone, page.Dataset.Timezone)} + } + if !reflect.DeepEqual(d.SourceColumns, page.Dataset.SourceColumns) { + return &SchemaChangedError{Reason: "fields, field IDs, field types, or field order changed"} + } + d.Records = append(d.Records, page.Dataset.Records...) + return nil +} + +func sourceColumn(name, fieldID, fieldType string) (Column, error) { + if _, ok := specForFieldType(fieldType); !ok { + return Column{}, &MatrixError{Reason: fmt.Sprintf("field %q has unsupported field type %q", name, fieldType)} + } + return Column{Name: name, FieldID: fieldID, FieldType: fieldType}, nil +} + +func normalizeCell(column Column, value any, timezone string) (any, error) { + spec, ok := specForFieldType(column.FieldType) + if !ok { + return nil, newDetailErrorf("unsupported field type %q", column.FieldType) + } + if spec.repeated { + if value == nil { + return []any{}, nil + } + items, ok := value.([]any) + if !ok { + return nil, newDetailErrorf("expected array, got %T", value) + } + for index, item := range items { + switch spec.kind { + case KindString: + if _, ok := item.(string); !ok { + return nil, newDetailErrorf("array item %d must be a string, got %T", index+1, item) + } + case KindObject: + if _, ok := item.(map[string]any); !ok { + return nil, newDetailErrorf("array item %d must be an object, got %T", index+1, item) + } + } + } + return items, nil + } + if value == nil { + if column.FieldType == "checkbox" { + return false, nil + } + return nil, nil + } + switch spec.kind { + case KindString: + text, ok := value.(string) + if !ok { + return nil, newDetailErrorf("expected string, got %T", value) + } + if column.FieldType == "datetime" || column.FieldType == "created_at" || column.FieldType == "updated_at" { + return normalizeDateTime(text, timezone) + } + return text, nil + case KindNumber: + switch value.(type) { + case json.Number, float64, float32, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: + return value, nil + default: + return nil, newDetailErrorf("expected number, got %T", value) + } + case KindBoolean: + if _, ok := value.(bool); !ok { + return nil, newDetailErrorf("expected boolean, got %T", value) + } + return value, nil + case KindObject: + if _, ok := value.(map[string]any); !ok { + return nil, newDetailErrorf("expected object, got %T", value) + } + return value, nil + default: + return nil, newDetailErrorf("unsupported value kind %q", spec.kind) + } +} + +func normalizeDateTime(value, timezone string) (string, error) { + if _, err := time.Parse(time.RFC3339Nano, value); err == nil { + return value, nil + } + location, err := time.LoadLocation(timezone) + if err != nil { + return "", wrapDetailErrorf(err, "invalid timezone %q", timezone) + } + parsed, err := time.ParseInLocation("2006-01-02 15:04:05", value, location) + if err != nil { + return "", newDetailErrorf("expected YYYY-MM-DD HH:mm:ss or RFC3339, got %q", value) + } + return parsed.Format(time.RFC3339), nil +} + +func stringList(raw any, name string) ([]string, error) { + items, ok := raw.([]any) + if !ok { + return nil, &MatrixError{Reason: name + " must be an array of strings"} + } + values := make([]string, 0, len(items)) + for index, item := range items { + value, ok := item.(string) + if !ok { + return nil, &MatrixError{Reason: fmt.Sprintf("%s item %d must be a string", name, index+1)} + } + values = append(values, value) + } + return values, nil +} + +func ignoredFieldList(raw any) ([]IgnoredField, error) { + items, ok := raw.([]any) + if !ok { + return nil, &MatrixError{Reason: "ignored_fields must be an array"} + } + fields := make([]IgnoredField, 0, len(items)) + for index, item := range items { + object, ok := item.(map[string]any) + if !ok { + return nil, &MatrixError{Reason: fmt.Sprintf("ignored_fields item %d must be an object", index+1)} + } + field := IgnoredField{} + var valid bool + field.ID, valid = object["id"].(string) + if !valid { + return nil, &MatrixError{Reason: fmt.Sprintf("ignored_fields item %d id must be a string", index+1)} + } + field.Name, valid = object["name"].(string) + if !valid { + return nil, &MatrixError{Reason: fmt.Sprintf("ignored_fields item %d name must be a string", index+1)} + } + field.Reason, valid = object["reason"].(string) + if !valid { + return nil, &MatrixError{Reason: fmt.Sprintf("ignored_fields item %d reason must be a string", index+1)} + } + fields = append(fields, field) + } + return fields, nil +} diff --git a/shortcuts/base/recordexport/dataset_test.go b/shortcuts/base/recordexport/dataset_test.go new file mode 100644 index 0000000000..a3f3339c04 --- /dev/null +++ b/shortcuts/base/recordexport/dataset_test.go @@ -0,0 +1,338 @@ +// Copyright (c) 2026 Lark Technologies Pte. Ltd. +// SPDX-License-Identifier: MIT + +package recordexport + +import ( + "bytes" + "encoding/json" + "strings" + "testing" +) + +func TestParseMatrixNormalizesTypedRows(t *testing.T) { + page, err := ParseMatrix(map[string]any{ + "rev": json.Number("42"), + "timezone": "Asia/Shanghai", + "fields": []any{"Name", "Tags", "Owner", "Due", "Score", "Done", "Place", "Formula"}, + "field_id_list": []any{"fld_name", "fld_tags", "fld_owner", "fld_due", "fld_score", "fld_done", "fld_place", "fld_formula"}, + "field_type_list": []any{ + "text", "select", "user", "datetime", "number", "checkbox", "location", "formula", + }, + "record_id_list": []any{"rec_1", "rec_2"}, + "data": []any{ + []any{"Alice", nil, nil, "2026-08-04 12:30:00", 12.5, false, map[string]any{"lng": 1.0, "lat": 2.0, "full_address": "北京市"}, "ok"}, + []any{nil, []any{"P0"}, []any{map[string]any{"id": "ou_1", "name": "Bob"}}, nil, nil, nil, nil, nil}, + }, + "has_more": true, + }) + if err != nil { + t.Fatalf("ParseMatrix() error = %v", err) + } + if !page.HasMore { + t.Fatal("HasMore = false, want true") + } + if page.Rev == nil || *page.Rev != 42 { + t.Fatalf("Rev = %#v, want 42", page.Rev) + } + if got := page.Dataset.Columns[0].PhysicalType(); got != "string" { + t.Fatalf("record_id physical type = %q", got) + } + if got := page.Dataset.Columns[2].PhysicalType(); got != "array" { + t.Fatalf("Tags physical type = %q", got) + } + if got := page.Dataset.Columns[3].PhysicalType(); got != "array>" { + t.Fatalf("Owner physical type = %q", got) + } + if got := page.Dataset.Columns[4].PhysicalType(); got != "string|null" { + t.Fatalf("Due physical type = %q", got) + } + if got := page.Dataset.Columns[6].PhysicalType(); got != "boolean" { + t.Fatalf("Done physical type = %q", got) + } + if got := page.Dataset.Columns[7].PhysicalType(); got != "struct|null" { + t.Fatalf("Place physical type = %q", got) + } + first := page.Dataset.Records[0].Values + if tags, ok := first[2].([]any); !ok || len(tags) != 0 { + t.Fatalf("empty Tags = %#v, want []", first[2]) + } + if owners, ok := first[3].([]any); !ok || len(owners) != 0 { + t.Fatalf("empty Owner = %#v, want []", first[3]) + } + if got := first[4]; got != "2026-08-04T12:30:00+08:00" { + t.Fatalf("Due = %#v", got) + } + if got := first[6]; got != false { + t.Fatalf("Done = %#v, want false", got) + } + place := first[7].(map[string]any) + if got := place["full_address"]; got != "北京市" { + t.Fatalf("Place.full_address = %#v", got) + } + second := page.Dataset.Records[1].Values + if second[1] != nil || second[4] != nil || second[5] != nil { + t.Fatalf("nullable scalars = %#v", second) + } + if got := second[6]; got != false { + t.Fatalf("empty Done = %#v, want false", got) + } + + var output bytes.Buffer + if err := WriteNDJSON(&output, page.Dataset); err != nil { + t.Fatal(err) + } + lines := bytes.Split(bytes.TrimSpace(output.Bytes()), []byte("\n")) + var secondRow map[string]any + if err := json.Unmarshal(lines[1], &secondRow); err != nil { + t.Fatal(err) + } + if got := secondRow["Done"]; got != false { + t.Fatalf("NDJSON Done = %#v, want false", got) + } +} + +func TestNormalizeDateTimePreservesUpstreamOffset(t *testing.T) { + value := "2026-11-01T01:30:00.123456-04:00" + got, err := normalizeDateTime(value, "America/New_York") + if err != nil { + t.Fatalf("normalizeDateTime() error = %v", err) + } + if got != value { + t.Fatalf("normalizeDateTime() = %q, want unchanged %q", got, value) + } +} + +func TestManifestDeclaresKnownObjectStructsWithoutRewritingRows(t *testing.T) { + page, err := ParseMatrix(map[string]any{ + "timezone": "UTC", + "fields": []any{ + "Users", "Chats", "Links", "Files", "Creator", "Updater", + }, + "field_id_list": []any{ + "fld_users", "fld_chats", "fld_links", "fld_files", "fld_creator", "fld_updater", + }, + "field_type_list": []any{ + "user", "group_chat", "link", "attachment", "created_by", "updated_by", + }, + "record_id_list": []any{"rec_1"}, + "data": []any{[]any{ + []any{map[string]any{"id": "ou_1"}}, + []any{map[string]any{"id": "oc_1"}}, + []any{map[string]any{"id": "rec_link"}}, + []any{map[string]any{"file_token": "box_1"}}, + []any{map[string]any{"id": "ou_creator"}}, + []any{map[string]any{"id": "ou_updater"}}, + }}, + }) + if err != nil { + t.Fatal(err) + } + + wantTypes := map[string]string{ + "Users": "array>", + "Chats": "array>", + "Links": "array>", + "Files": "array>", + "Creator": "array>", + "Updater": "array>", + } + manifest := BuildManifest(page.Dataset, ManifestOptions{ + BaseToken: "base_x", TableID: "tbl_x", PageCount: 1, + RecordFile: "/tmp/out.ndjson", ManifestFile: "/tmp/out.manifest.json", + }) + for name, want := range wantTypes { + if got := manifest.Columns[name].PhysicalType; got != want { + t.Errorf("%s physical type = %q, want %q", name, got, want) + } + } + + users := page.Dataset.Records[0].Values[1].([]any) + if _, exists := users[0].(map[string]any)["name"]; exists { + t.Fatalf("runtime Users value was rewritten: %#v", users) + } + files := page.Dataset.Records[0].Values[4].([]any) + if _, exists := files[0].(map[string]any)["size"]; exists { + t.Fatalf("runtime Files value was rewritten: %#v", files) + } +} + +func TestParseMatrixSystemRecordIDWins(t *testing.T) { + page, err := ParseMatrix(map[string]any{ + "timezone": "UTC", + "fields": []any{"record_id", "Name"}, + "field_id_list": []any{"fld_shadow", "fld_name"}, + "field_type_list": []any{"text", "text"}, + "record_id_list": []any{"rec_real"}, + "data": []any{[]any{"user-value", "Alice"}}, + }) + if err != nil { + t.Fatalf("ParseMatrix() error = %v", err) + } + if len(page.Dataset.Columns) != 2 || page.Dataset.Columns[0].Name != "record_id" || page.Dataset.Columns[1].Name != "Name" { + t.Fatalf("export columns = %#v", page.Dataset.Columns) + } + if got := page.Dataset.Records[0].Values[0]; got != "rec_real" { + t.Fatalf("record_id = %#v", got) + } +} + +func TestDatasetAppendRejectsSchemaChange(t *testing.T) { + first, err := ParseMatrix(matrixFixture("Name", "fld_name", "text", "rec_1", "Alice")) + if err != nil { + t.Fatal(err) + } + second, err := ParseMatrix(matrixFixture("Name", "fld_other", "text", "rec_2", "Bob")) + if err != nil { + t.Fatal(err) + } + dataset := first.Dataset + if err := dataset.AppendPage(second); err == nil { + t.Fatal("AppendPage() error = nil, want schema change") + } +} + +func TestManifestExamplesAndNDJSON(t *testing.T) { + longText := strings.Repeat("长", 150) + page, err := ParseMatrix(map[string]any{ + "timezone": "UTC", + "fields": []any{"Long", "EmptyUsers", "Count"}, + "field_id_list": []any{"fld_long", "fld_empty", "fld_count"}, + "field_type_list": []any{"text", "user", "number"}, + "record_id_list": []any{"rec_1"}, + "data": []any{[]any{longText, nil, 0}}, + }) + if err != nil { + t.Fatal(err) + } + manifest := BuildManifest(page.Dataset, ManifestOptions{ + BaseToken: "base_x", TableID: "tbl_x", PageCount: 1, + RecordFile: "/tmp/out.ndjson", ManifestFile: "/tmp/out.manifest.json", + }) + if !manifest.Columns["Long"].ExampleTruncated { + t.Fatal("Long.example_truncated = false") + } + if manifest.Columns["EmptyUsers"].Example != nil || manifest.Columns["EmptyUsers"].Hint == "" { + t.Fatalf("EmptyUsers metadata = %#v", manifest.Columns["EmptyUsers"]) + } + if got := manifest.Columns["EmptyUsers"].PhysicalType; got != "array>" { + t.Fatalf("EmptyUsers physical type = %q", got) + } + if got := manifest.Columns["Count"].Example; got != 0 { + t.Fatalf("Count.example = %#v, want 0", got) + } + if got := *manifest.Columns["Long"].Stats.MaxLength; got != 150 { + t.Fatalf("Long.stats.max_length = %d, want 150", got) + } + if got := *manifest.Columns["EmptyUsers"].Stats.EmptyCount; got != 1 { + t.Fatalf("EmptyUsers.stats.empty_count = %d, want 1", got) + } + if got := *manifest.Columns["Count"].Stats.Avg; got != 0 { + t.Fatalf("Count.stats.avg = %v, want 0", got) + } + + var output bytes.Buffer + if err := WriteNDJSON(&output, page.Dataset); err != nil { + t.Fatal(err) + } + var row map[string]any + if err := json.Unmarshal(bytes.TrimSpace(output.Bytes()), &row); err != nil { + t.Fatalf("unmarshal ndjson: %v\n%s", err, output.String()) + } + if row["record_id"] != "rec_1" { + t.Fatalf("row = %#v", row) + } + if empty, ok := row["EmptyUsers"].([]any); !ok || len(empty) != 0 { + t.Fatalf("EmptyUsers = %#v", row["EmptyUsers"]) + } +} + +func TestManifestColumnStatsByFieldType(t *testing.T) { + page, err := ParseMatrix(map[string]any{ + "timezone": "UTC", + "fields": []any{ + "Text", "Number", "When", "Done", "Place", "Tags", + }, + "field_id_list": []any{ + "fld_text", "fld_number", "fld_when", "fld_done", "fld_place", "fld_tags", + }, + "field_type_list": []any{ + "text", "number", "datetime", "checkbox", "location", "select", + }, + "record_id_list": []any{"rec_1", "rec_22", "rec_333"}, + "data": []any{ + []any{"短", 10, "2026-08-01T10:00:00+08:00", true, map[string]any{"lng": 1.0, "lat": 2.0, "full_address": "A"}, []any{"a", "b"}}, + []any{"最长值", 20.0, "2026-08-03T10:00:00+08:00", false, nil, nil}, + []any{nil, nil, nil, nil, nil, []any{"c"}}, + }, + }) + if err != nil { + t.Fatal(err) + } + + manifest := BuildManifest(page.Dataset, ManifestOptions{ + BaseToken: "base_x", TableID: "tbl_x", PageCount: 1, + RecordFile: "/tmp/out.ndjson", ManifestFile: "/tmp/out.manifest.json", + }) + + if got := *manifest.Columns["record_id"].Stats.MaxLength; got != 7 { + t.Errorf("record_id.stats.max_length = %d, want 7", got) + } + textStats := manifest.Columns["Text"].Stats + if *textStats.NullCount != 1 || *textStats.MaxLength != 3 { + t.Errorf("Text.stats = %#v", textStats) + } + numberStats := manifest.Columns["Number"].Stats + if *numberStats.NullCount != 1 || numberStats.Min != 10.0 || numberStats.Max != 20.0 || *numberStats.Avg != 15.0 { + t.Errorf("Number.stats = %#v", numberStats) + } + whenStats := manifest.Columns["When"].Stats + if *whenStats.NullCount != 1 || whenStats.Min != "2026-08-01T10:00:00+08:00" || whenStats.Max != "2026-08-03T10:00:00+08:00" { + t.Errorf("When.stats = %#v", whenStats) + } + doneStats := manifest.Columns["Done"].Stats + if *doneStats.TrueCount != 1 || doneStats.NullCount != nil { + t.Errorf("Done.stats = %#v", doneStats) + } + placeStats := manifest.Columns["Place"].Stats + if *placeStats.NullCount != 2 { + t.Errorf("Place.stats = %#v", placeStats) + } + tagStats := manifest.Columns["Tags"].Stats + if *tagStats.EmptyCount != 1 || *tagStats.MaxLength != 2 || *tagStats.AvgLength != 1 { + t.Errorf("Tags.stats = %#v", tagStats) + } +} + +func TestParseMatrixRejectsParallelArrayMismatch(t *testing.T) { + _, err := ParseMatrix(map[string]any{ + "timezone": "UTC", + "fields": []any{"Name"}, + "field_id_list": []any{}, + "field_type_list": []any{"text"}, + "record_id_list": []any{}, + "data": []any{}, + }) + if err == nil { + t.Fatal("ParseMatrix() error = nil") + } +} + +func TestParseMatrixRejectsInvalidRev(t *testing.T) { + fixture := matrixFixture("Name", "fld_name", "text", "rec_1", "Alice") + fixture["rev"] = "42" + if _, err := ParseMatrix(fixture); err == nil { + t.Fatal("ParseMatrix() error = nil, want invalid rev") + } +} + +func matrixFixture(name, id, fieldType, recordID string, value any) map[string]any { + return map[string]any{ + "timezone": "UTC", + "fields": []any{name}, + "field_id_list": []any{id}, + "field_type_list": []any{fieldType}, + "record_id_list": []any{recordID}, + "data": []any{[]any{value}}, + } +} diff --git a/shortcuts/base/recordexport/errors.go b/shortcuts/base/recordexport/errors.go new file mode 100644 index 0000000000..bf8e4ff0e2 --- /dev/null +++ b/shortcuts/base/recordexport/errors.go @@ -0,0 +1,30 @@ +// Copyright (c) 2026 Lark Technologies Pte. Ltd. +// SPDX-License-Identifier: MIT + +package recordexport + +import "fmt" + +// detailError is an intermediate conversion/encoding error. The Base command +// boundary classifies it as a typed invalid-response error before it can reach +// the user. +type detailError struct { + message string + cause error +} + +func (e *detailError) Error() string { return e.message } + +func (e *detailError) Unwrap() error { return e.cause } + +func newDetailErrorf(format string, args ...any) error { + return &detailError{message: fmt.Sprintf(format, args...)} +} + +func wrapDetailError(message string, cause error) error { + return &detailError{message: message + ": " + cause.Error(), cause: cause} +} + +func wrapDetailErrorf(cause error, format string, args ...any) error { + return wrapDetailError(fmt.Sprintf(format, args...), cause) +} diff --git a/shortcuts/base/recordexport/manifest.go b/shortcuts/base/recordexport/manifest.go new file mode 100644 index 0000000000..2d4fb48845 --- /dev/null +++ b/shortcuts/base/recordexport/manifest.go @@ -0,0 +1,412 @@ +// Copyright (c) 2026 Lark Technologies Pte. Ltd. +// SPDX-License-Identifier: MIT + +package recordexport + +import ( + "encoding/json" + "io" + "time" + "unicode/utf8" +) + +const ( + ManifestVersion = "v1" + FormatNDJSON = "ndjson" + emptyColumnHint = "All values are empty; type inference may be ambiguous. Avoid analyzing this column unless required." + maxExampleRunes = 128 + maxExampleItems = 3 +) + +// ColumnManifest is the user-facing description of one actual data column. +type ColumnManifest struct { + FieldID string `json:"field_id,omitempty"` + FieldType string `json:"field_type,omitempty"` + PhysicalType string `json:"physical_type"` + Stats ColumnStats `json:"stats"` + Example any `json:"example,omitempty"` + ExampleTruncated bool `json:"example_truncated,omitempty"` + Hint string `json:"hint,omitempty"` +} + +// ColumnStats contains only the metrics that are meaningful for a column's +// Base field type. All metrics describe the records in this export. +type ColumnStats struct { + NullCount *int `json:"null_count,omitempty"` + EmptyCount *int `json:"empty_count,omitempty"` + TrueCount *int `json:"true_count,omitempty"` + Min any `json:"min,omitempty"` + Max any `json:"max,omitempty"` + Avg *float64 `json:"avg,omitempty"` + MaxLength *int `json:"max_length,omitempty"` + AvgLength *float64 `json:"avg_length,omitempty"` +} + +// Manifest describes one exported artifact and the exact query boundary that +// produced it. Format is explicit so the same structure can describe future +// JSON-array and Parquet outputs. +type Manifest struct { + ManifestVersion string `json:"manifest_version"` + Format string `json:"format"` + BaseToken string `json:"base_token"` + TableID string `json:"table_id"` + Rev *int64 `json:"rev,omitempty"` + Timezone string `json:"timezone"` + QueryContext map[string]any `json:"query_context,omitempty"` + Offset int `json:"offset,omitempty"` + RequestedLimit int `json:"requested_limit,omitempty"` + RecordsCount int `json:"records_count"` + PageCount int `json:"page_count"` + HasMore bool `json:"has_more"` + NextOffset *int `json:"next_offset,omitempty"` + RecordFile string `json:"record_file"` + RecordFileSizeBytes int64 `json:"record_file_size_bytes"` + ManifestFile string `json:"manifest_file"` + Columns map[string]ColumnManifest `json:"columns"` + IgnoredFields []IgnoredField `json:"ignored_fields,omitempty"` + RecordNotFound []string `json:"record_not_found,omitempty"` +} + +// ManifestOptions carries query and file details that are outside Dataset. +type ManifestOptions struct { + BaseToken string + TableID string + Rev *int64 + QueryContext map[string]any + Offset int + RequestedLimit int + PageCount int + HasMore bool + RecordFile string + RecordFileSizeBytes int64 + ManifestFile string + IgnoredFields []IgnoredField + RecordNotFound []string +} + +// MinimalManifest is the stable low-token stdout result. +type MinimalManifest struct { + RecordFile string `json:"record_file"` + RecordFileSizeBytes int64 `json:"record_file_size_bytes"` + ManifestFile string `json:"manifest_file"` + RecordsCount int `json:"records_count"` + HasMore bool `json:"has_more"` +} + +func BuildManifest(dataset Dataset, opts ManifestOptions) Manifest { + manifest := Manifest{ + ManifestVersion: ManifestVersion, + Format: FormatNDJSON, + BaseToken: opts.BaseToken, + TableID: opts.TableID, + Rev: opts.Rev, + Timezone: dataset.Timezone, + QueryContext: opts.QueryContext, + Offset: opts.Offset, + RequestedLimit: opts.RequestedLimit, + RecordsCount: len(dataset.Records), + PageCount: opts.PageCount, + HasMore: opts.HasMore, + RecordFile: opts.RecordFile, + RecordFileSizeBytes: opts.RecordFileSizeBytes, + ManifestFile: opts.ManifestFile, + Columns: make(map[string]ColumnManifest, len(dataset.Columns)), + IgnoredFields: opts.IgnoredFields, + RecordNotFound: opts.RecordNotFound, + } + if opts.HasMore { + nextOffset := opts.Offset + len(dataset.Records) + manifest.NextOffset = &nextOffset + } + for columnIndex, column := range dataset.Columns { + metadata := ColumnManifest{ + FieldID: column.FieldID, + FieldType: column.FieldType, + PhysicalType: column.PhysicalType(), + Stats: buildColumnStats(dataset.Records, columnIndex, column), + } + if example, truncated, found := bestExample(dataset.Records, columnIndex); found { + metadata.Example = example + metadata.ExampleTruncated = truncated + } else { + metadata.Hint = emptyColumnHint + } + manifest.Columns[column.Name] = metadata + } + return manifest +} + +func buildColumnStats(records []Record, columnIndex int, column Column) ColumnStats { + if column.System { + maxLength := 0 + for _, record := range records { + if columnIndex >= len(record.Values) { + continue + } + value, ok := record.Values[columnIndex].(string) + if ok && utf8.RuneCountInString(value) > maxLength { + maxLength = utf8.RuneCountInString(value) + } + } + return ColumnStats{MaxLength: &maxLength} + } + + spec, ok := specForFieldType(column.FieldType) + if !ok { + return ColumnStats{} + } + if spec.repeated { + emptyCount, maxLength, totalLength := 0, 0, 0 + for _, record := range records { + if columnIndex >= len(record.Values) { + continue + } + items, ok := record.Values[columnIndex].([]any) + if !ok { + continue + } + length := len(items) + totalLength += length + if length == 0 { + emptyCount++ + } + if length > maxLength { + maxLength = length + } + } + avgLength := 0.0 + if len(records) > 0 { + avgLength = float64(totalLength) / float64(len(records)) + } + return ColumnStats{ + EmptyCount: &emptyCount, + MaxLength: &maxLength, + AvgLength: &avgLength, + } + } + + switch column.FieldType { + case "number": + nullCount, valueCount := 0, 0 + var minValue, maxValue, total float64 + for _, record := range records { + if columnIndex >= len(record.Values) || record.Values[columnIndex] == nil { + nullCount++ + continue + } + value, ok := numberAsFloat64(record.Values[columnIndex]) + if !ok { + continue + } + if valueCount == 0 || value < minValue { + minValue = value + } + if valueCount == 0 || value > maxValue { + maxValue = value + } + total += value + valueCount++ + } + stats := ColumnStats{NullCount: &nullCount} + if valueCount > 0 { + avg := total / float64(valueCount) + stats.Min, stats.Max, stats.Avg = minValue, maxValue, &avg + } + return stats + + case "datetime", "created_at", "updated_at": + nullCount := 0 + var minValue, maxValue string + var minTime, maxTime time.Time + found := false + for _, record := range records { + if columnIndex >= len(record.Values) || record.Values[columnIndex] == nil { + nullCount++ + continue + } + value, ok := record.Values[columnIndex].(string) + if !ok { + continue + } + parsed, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + continue + } + if !found || parsed.Before(minTime) { + minValue, minTime = value, parsed + } + if !found || parsed.After(maxTime) { + maxValue, maxTime = value, parsed + } + found = true + } + stats := ColumnStats{NullCount: &nullCount} + if found { + stats.Min, stats.Max = minValue, maxValue + } + return stats + + case "checkbox": + trueCount := 0 + for _, record := range records { + if columnIndex < len(record.Values) && record.Values[columnIndex] == true { + trueCount++ + } + } + return ColumnStats{TrueCount: &trueCount} + + case "location": + nullCount := 0 + for _, record := range records { + if columnIndex >= len(record.Values) || record.Values[columnIndex] == nil { + nullCount++ + } + } + return ColumnStats{NullCount: &nullCount} + + default: + nullCount, maxLength := 0, 0 + for _, record := range records { + if columnIndex >= len(record.Values) || record.Values[columnIndex] == nil { + nullCount++ + continue + } + value, ok := record.Values[columnIndex].(string) + if ok && utf8.RuneCountInString(value) > maxLength { + maxLength = utf8.RuneCountInString(value) + } + } + return ColumnStats{NullCount: &nullCount, MaxLength: &maxLength} + } +} + +func numberAsFloat64(value any) (float64, bool) { + switch typed := value.(type) { + case json.Number: + converted, err := typed.Float64() + return converted, err == nil + case float64: + return typed, true + case float32: + return float64(typed), true + case int: + return float64(typed), true + case int8: + return float64(typed), true + case int16: + return float64(typed), true + case int32: + return float64(typed), true + case int64: + return float64(typed), true + case uint: + return float64(typed), true + case uint8: + return float64(typed), true + case uint16: + return float64(typed), true + case uint32: + return float64(typed), true + case uint64: + return float64(typed), true + default: + return 0, false + } +} + +func (m Manifest) Minimal() MinimalManifest { + return MinimalManifest{ + RecordFile: m.RecordFile, RecordFileSizeBytes: m.RecordFileSizeBytes, + ManifestFile: m.ManifestFile, + RecordsCount: m.RecordsCount, HasMore: m.HasMore, + } +} + +// WriteManifest writes deterministic indented JSON without HTML escaping. +func WriteManifest(w io.Writer, manifest Manifest) error { + encoder := json.NewEncoder(w) + encoder.SetEscapeHTML(false) + encoder.SetIndent("", " ") + if err := encoder.Encode(manifest); err != nil { + return wrapDetailError("encode manifest", err) + } + return nil +} + +func bestExample(records []Record, columnIndex int) (any, bool, bool) { + var best any + bestSize := 0 + bestTruncated := false + found := false + for _, record := range records { + if columnIndex >= len(record.Values) { + continue + } + value := record.Values[columnIndex] + if isEmptyExample(value) { + continue + } + encoded, err := json.Marshal(value) + if err != nil { + continue + } + if found && len(encoded) >= bestSize { + continue + } + best, bestTruncated = truncateExample(value) + bestSize = len(encoded) + found = true + } + return best, bestTruncated, found +} + +func isEmptyExample(value any) bool { + switch typed := value.(type) { + case nil: + return true + case string: + return typed == "" + case []any: + return len(typed) == 0 + case map[string]any: + return len(typed) == 0 + default: + return false + } +} + +func truncateExample(value any) (any, bool) { + switch typed := value.(type) { + case string: + if utf8.RuneCountInString(typed) <= maxExampleRunes { + return typed, false + } + runes := []rune(typed) + return string(runes[:maxExampleRunes]), true + case []any: + limit := len(typed) + truncated := false + if limit > maxExampleItems { + limit = maxExampleItems + truncated = true + } + items := make([]any, 0, limit) + for _, item := range typed[:limit] { + value, itemTruncated := truncateExample(item) + items = append(items, value) + truncated = truncated || itemTruncated + } + return items, truncated + case map[string]any: + object := make(map[string]any, len(typed)) + truncated := false + for key, item := range typed { + value, itemTruncated := truncateExample(item) + object[key] = value + truncated = truncated || itemTruncated + } + return object, truncated + default: + return value, false + } +} diff --git a/shortcuts/base/recordexport/ndjson.go b/shortcuts/base/recordexport/ndjson.go new file mode 100644 index 0000000000..c99f8399d6 --- /dev/null +++ b/shortcuts/base/recordexport/ndjson.go @@ -0,0 +1,33 @@ +// Copyright (c) 2026 Lark Technologies Pte. Ltd. +// SPDX-License-Identifier: MIT + +package recordexport + +import ( + "encoding/json" + "io" +) + +// WriteNDJSON writes one complete object per line. Values are materialized by +// column name here so future exporters can consume Dataset without inheriting +// NDJSON-specific maps. +func WriteNDJSON(w io.Writer, dataset Dataset) error { + encoder := json.NewEncoder(w) + encoder.SetEscapeHTML(false) + for rowIndex, record := range dataset.Records { + if len(record.Values) != len(dataset.Columns) { + return newDetailErrorf( + "record %d has %d values; dataset schema has %d columns", + rowIndex+1, len(record.Values), len(dataset.Columns), + ) + } + object := make(map[string]any, len(dataset.Columns)) + for index, column := range dataset.Columns { + object[column.Name] = record.Values[index] + } + if err := encoder.Encode(object); err != nil { + return wrapDetailErrorf(err, "encode record %d", rowIndex+1) + } + } + return nil +} diff --git a/shortcuts/common/runner.go b/shortcuts/common/runner.go index c6a5ed8e36..3cbe5adceb 100644 --- a/shortcuts/common/runner.go +++ b/shortcuts/common/runner.go @@ -969,6 +969,10 @@ func runShortcut(cmd *cobra.Command, f *cmdutil.Factory, s *Shortcut, botOnly bo if err := s.Normalize(rctx.ctx, flagContext); err != nil { return attributeAliasValidationError(rctx, err) } + // Normalize may canonicalize --format from another strong user signal + // (for example, an artifact --output path). Keep the cached value used by + // output and jq validation aligned with the canonical flag. + rctx.Format = rctx.Str("format") } if err := validateEnumFlags(rctx, s.Flags); err != nil { return attributeAliasValidationError(rctx, err) @@ -978,8 +982,14 @@ func runShortcut(cmd *cobra.Command, f *cmdutil.Factory, s *Shortcut, botOnly bo return attributeAliasValidationError(rctx, err) } } - if err := output.ValidateJqFlags(rctx.JqExpr, "", rctx.Format); err != nil { - return err + if rctx.JqExpr != "" && slices.Contains(s.JQFormats, rctx.Format) { + if err := output.ValidateJqExpression(rctx.JqExpr); err != nil { + return err + } + } else { + if err := output.ValidateJqFlags(rctx.JqExpr, "", rctx.Format); err != nil { + return err + } } if s.Validate != nil { if err := s.Validate(rctx.ctx, rctx); err != nil { diff --git a/shortcuts/common/types.go b/shortcuts/common/types.go index 02bc04dd50..c1f6a1e5c5 100644 --- a/shortcuts/common/types.go +++ b/shortcuts/common/types.go @@ -53,6 +53,10 @@ type Shortcut struct { HasFormat bool // Deprecated: --format is now always injected; this field has no effect. Tips []string // optional tips shown in --help output Hidden bool // hide from --help / tab completion (still executable); use when deprecating a command in favor of a replacement + // JQFormats allows a shortcut-owned artifact format to keep stdout as JSON + // and apply --jq to that JSON value. Ordinary record-stream formats remain + // governed by the global JSON-only jq rule. + JQFormats []string // Business logic hooks. // Normalize is the business-owned compatibility stage inside shortcut diff --git a/skills/lark-base/SKILL.md b/skills/lark-base/SKILL.md index e040bab83b..e89f1436b7 100644 --- a/skills/lark-base/SKILL.md +++ b/skills/lark-base/SKILL.md @@ -1,6 +1,6 @@ --- name: lark-base -version: 1.2.4 +version: 1.2.5 description: "飞书多维表格（Base）操作：建表、字段、记录、视图、统计、公式/lookup、表单、仪表盘、workflow、角色权限；遇到 Base/多维表格/bitable 或 /base/ 链接时使用。文件导入/导出转 lark-drive，认证/授权转 lark-shared。" metadata: requires: @@ -31,8 +31,9 @@ metadata: - Base 业务操作只使用 `lark-cli base +...` shortcut，不使用旧聚合式 `+table / +field / +record / +view / +history / +workspace`。 - 执行 update 前必须先查当前 shortcut 的 `--help` 或对应 reference。若命令要求完整配置，首次请求必须基于可信的当前配置执行 read-modify-write：只修改用户明确指定的内容，保留其他仍适用的可写配置，并按命令要求的结构提交。若命令支持局部／delta update，按其契约提交最小合法 payload；不得以不完整请求试错补参。 - Base CLI/OpenAPI 当前不支持视图行高、冻结列、列宽等 UI-only 外观设置。遇到这类需求，说明能力边界并停止，不要猜测未文档化参数或改走 raw API。 -- 本地文件与 Base 之间的导入/导出转 `lark-drive`，具体格式、参数、路径限制和仅结构导出规则由 `lark-drive` 负责；导入完成后再回到 Base 命令。 -- 在线复制 Base 使用 `+base-copy`，不要绕行导出/导入。 +- **高频：数据分析。** 记录作为分析、解析、比较或可复用的本地输入时，首选 `--output .ndjson --minimal-stdout`，并按 [Base 数据表查询与分析 SOP](references/lark-base-data-analysis-sop.md) 处理。 +- **低频：在线复制。** 复制整个 Base 使用 `+base-copy`，复制 Base 内单张数据表使用 `+table-copy`。 +- **更低频：文件导入/导出。** 本地文件与 Base 之间的导入/导出转 `lark-drive`；具体格式、参数、路径限制和仅结构导出规则由 `lark-drive` 负责，导入完成后再回到 Base 命令。 - 认证、初始化、scope、身份切换、权限不足恢复属于 `lark-shared`；Base 文档只保留会影响 Base 路径选择的权限规则。 ## 先获取 Base Token 和所需 ID @@ -55,24 +56,24 @@ metadata: | 查看 Base 内资源目录 | `+base-block-list` | 想先了解一个 Base 里有哪些 table/docx/dashboard/workflow/folder 时优先用它；返回 ID 关系和 fewshot 看 `--help` | | 管理 Base 内资源目录 | `+base-block-create/move/rename/delete` | 创建或整理 Base 直接管理的 folder/table/docx/dashboard/workflow；资源内容继续用对应命令 | | 管理数据表 | `+table-list/get/create/update/delete` | 处理 table 的列出、详情、创建、重命名和删除 | -| 复制 Base 内单张数据表 | `+table-copy` / `+table-copy-status` | 默认只复制结构；只有用户明确要求复制全表、数据、行或记录时才传 `--range all`；异步任务按返回的 `task_id` 查询或续等 | +| 复制 Base 内单张数据表 | `+table-copy` / `+table-copy-status` | 在线复制单张数据表；复制范围和异步任务参数查看 `--help` | | 列/查/删字段 | `+field-list/get/delete/search-options` | 写入前用 list/get 确认字段类型、选项、ID；删除前确认目标字段 | | 创建/更新字段 | `+field-create` / `+field-update` | 同一表创建多个字段时，默认一次向 `+field-create --json` 传字段对象数组；预计串行运行时间超过 caller/tool timeout 时按时间预算拆分，不按固定条数切块；仅创建一个或多个只含 `name` + `type:text` 的简单字段时按 `+field-create --help` 即可，其他类型或属性必读 [lark-base-field-json.md](references/lark-base-field-json.md)；公式读 [formula-field-guide.md](references/formula-field-guide.md)，lookup 读 [lookup-field-guide.md](references/lookup-field-guide.md)；仍需逐项恢复或命令细节时读 [lark-base-field-create.md](references/lark-base-field-create.md)，更新细节读 [lark-base-field-update.md](references/lark-base-field-update.md) | -| 读记录明细 | `+record-get` / `+record-list` / `+record-search` | 涉及筛选、排序、Top/Bottom N、聚合、多表关联、全局结论时读 [lark-base-data-analysis-sop.md](references/lark-base-data-analysis-sop.md) | +| 读取已知记录 | `+record-get` | 已知具体 `record_id` 时可以直接读取记录 | +| 查询或分析数据表记录 | 由 [Base 数据表查询与分析 SOP](references/lark-base-data-analysis-sop.md) 选择 | 数据表记录查询和分析任务先读 SOP | | 写记录 | `+record-upsert` / `+record-batch-create` / `+record-batch-update` | 必读 [lark-base-record-upsert.md](references/lark-base-record-upsert.md) / [lark-base-record-batch-create.md](references/lark-base-record-batch-create.md) / [lark-base-record-batch-update.md](references/lark-base-record-batch-update.md) 和 [lark-base-cell-value.md](references/lark-base-cell-value.md) | -| 附件字段 | `+record-upload-attachment` / `+record-download-attachment` / `+record-remove-attachment` | 附件不要伪造成普通 CellValue；上传走本地文件，下载/删除按 file token 或字段定位 | +| 附件字段 | `+record-upload-attachment` / `+record-download-attachment` / `+record-remove-attachment` | 使用附件操作命令上传本地文件系统中的文件，下载/删除按 file token 或字段定位 | | 删除记录 / 分享记录链接 / 历史 | `+record-delete` / `+record-share-link-create` / `+record-history-list` | 删除前确认 record；分享链接最多 100 条；历史读 [lark-base-record-history-list.md](references/lark-base-record-history-list.md)，只查单条记录，不做整表审计 | | 管理视图 | `+view-*` | `+view-set-filter` 读 [lark-base-view-set-filter.md](references/lark-base-view-set-filter.md)（filter 条件结构见公共协议 [lark-base-filter-condition.md](references/lark-base-filter-condition.md)）；其余配置先 get 现状，再按返回结构更新 | -| 一次性聚合统计 | `+data-query` | 必读 [lark-base-data-analysis-sop.md](references/lark-base-data-analysis-sop.md) 和入口 [lark-base-data-query-guide.md](references/lark-base-data-query-guide.md)；完整 DSL 再读 [lark-base-data-query.md](references/lark-base-data-query.md) | | 公式字段 | `+field-create/update --json '{"type":"formula",...}'` | 必读 [formula-field-guide.md](references/formula-field-guide.md)，读后再加隐藏确认 flag `--i-have-read-guide` | | Lookup 字段 | `+field-create/update --json '{"type":"lookup",...}'` | 必读 [lookup-field-guide.md](references/lookup-field-guide.md)，读后再加隐藏确认 flag `--i-have-read-guide` | | 表单提交 | `+form-submit` | 先读 [lark-base-form-detail.md](references/lark-base-form-detail.md) 获取题目、filter 和附件所需 `base_token`；提交 JSON 读 [lark-base-form-submit.md](references/lark-base-form-submit.md) | | 表单题目创建/更新 | `+form-questions-create` / `+form-questions-update` | Base 内表单按 table 管理；先确定并复用真实 `table_id`。读 [lark-base-form-questions-create.md](references/lark-base-form-questions-create.md) / [lark-base-form-questions-update.md](references/lark-base-form-questions-update.md)；题目显隐条件 `visible_rule` 结构见公共协议 [lark-base-filter-condition.md](references/lark-base-filter-condition.md) | | Base 内表单管理 | `+form-list/get/create/update/delete` / `+form-questions-list/delete` | 缺少或不确定归属时，先用 `+table-list` 或 `+base-block-list` 取得真实 `table_id`；这些命令使用 `--base-token + --table-id` 并在整个工作流中复用同一 `table_id`，删除前确认目标表单 | -| 分享表单详情 | `+form-detail --share-token ` | 只接受表单分享链接里的 `share_token`，不要传 `--base-token` / `--form-id`；提交前读 [lark-base-form-detail.md](references/lark-base-form-detail.md) | +| 分享表单详情 | `+form-detail --share-token ` | 使用表单分享链接里的 `share_token`；提交前读 [lark-base-form-detail.md](references/lark-base-form-detail.md) | | 仪表盘与组件 | `+dashboard-*` / `+dashboard-block-*` | 提到图表/看板/block 时先读 [lark-base-dashboard.md](references/lark-base-dashboard.md)；组件 `data_config` 读 [dashboard-block-data-config.md](references/dashboard-block-data-config.md)；读取一个或多个图表计算结果用 `+dashboard-block-get-data`；读取完整仪表盘时按 block 类型分流，文本和不支持直接取数的图表按 reference 恢复 | | Workflow | `+workflow-*` | 创建/更新或理解 steps 时读入口 [lark-base-workflow-guide.md](references/lark-base-workflow-guide.md) 和 steps JSON SSOT [lark-base-workflow-schema.md](references/lark-base-workflow-schema.md)；list/get/enable/disable 只处理 workflow ID 与启停状态 | -| 高级权限与角色 | `+advperm-*` / `+role-*` | 角色操作先读入口 [lark-base-role-guide.md](references/lark-base-role-guide.md)；角色 create/update 或解读完整配置再读权限 JSON SSOT [role-config.md](references/role-config.md)；系统角色不可删除；关闭高级权限会影响自定义角色 | +| 高级权限与角色 | `+advperm-*` / `+role-*` | 角色操作先读入口 [lark-base-role-guide.md](references/lark-base-role-guide.md)；角色 create/update 或解读完整配置再读权限 JSON SSOT [role-config.md](references/role-config.md)；关闭高级权限会影响自定义角色 | ## Base 心智模型 @@ -81,13 +82,10 @@ metadata: - `base-block` 只负责资源目录管理，包括创建资源、移动到 folder、重命名和删除；具体资源内容仍走 table/dashboard/workflow 命令。 - 新建 Base 时，强烈推荐一次性执行 `lark-cli base +base-create --name "" --table-name "" --fields ''`，同时配置新 Base 里唯一一个初始数据表的 name 和 schema；使用 `--fields` 前先读 [lark-base-field-json.md](references/lark-base-field-json.md) 或复用 `+field-create` 的字段 JSON 形状，不要猜字段属性。 - `+base-create` 不传 `--table-name` 和 `--fields` 时，会创建一个默认 schema 的初始数据表。 -- `+table-copy` 的安全默认值是只复制表结构；用户没有明确要求记录时省略 `--range`，明确要求包含记录时才传 `--range all`。`--table-id` 可直接使用当前 Base 中的表 ID 或表名。 +- `+table-copy` 用于在线复制 Base 内的数据表，`--table-id` 可使用当前 Base 中的表 ID 或表名；复制范围等参数查看 `--help`。 - 表、字段、视图、workflow、dashboard block 的名称和 ID 必须来自真实返回，不要凭用户口述猜。 -- 存储字段可写；系统字段、`formula`、`lookup` 只读；附件字段走专用 attachment 命令。 -- 一次性原始记录查询优先用 `+record-list` / `+record-search` 的 filter/sort；聚合分析优先用 `+data-query`；需要长期显示在表中时，才新增 `formula` / `lookup` 字段。 - `formula` 适合常规计算、条件判断、文本/日期处理和长期派生指标；`lookup` 适合明确的跨表查找、筛选后取值或聚合引用。 -- 写入、分析、公式、lookup、workflow、dashboard 前，先读取真实结构：表、字段、视图、关联表和 dashboard block 名称都以命令返回为准。 -- 跨表场景必须读取目标表结构；link 单元格中的关联 `record_id` 只是连接键，最终回答要回查并展示用户可读字段。 +- 写入、公式、lookup、workflow、dashboard 前，先读取真实结构：表、字段、视图、关联表和 dashboard block 名称都以命令返回为准。 ## 身份与权限降级 @@ -98,18 +96,6 @@ metadata: - `91403` 或明确不可访问错误不要循环换身份重试。 - `+base-create` / `+base-copy` 若用 bot 身份执行，关注返回中的 `permission_grant`，并把用户是否可打开新 Base 告知用户。 -## 查询与统计规则 - -涉及查询、统计或判断结论时，先阅读 [lark-base-data-analysis-sop.md](references/lark-base-data-analysis-sop.md)，并遵守： - -1. `+record-list` 的默认页、固定 `--limit` 和本地 `jq` 只能证明已读取范围内的事实，不能直接支撑全局最值、全量计数、Top/Bottom N、异常识别或分组结论。 -2. 能由 Base 表达的筛选、排序、投影、聚合、分组和限制，应在 Base 云端查询能力中执行；不要先拉原始记录到本地上下文再手工筛选排序。 -3. `has_more=true` 或等价分页信号表示当前结果不是全量；除非用户只要样例/前 N 条，不能基于该页回答全局问题。 -4. 多表查询必须先确认关系字段和连接键；link 单元格里的 `record_id` 是关系键，不是用户可读答案。 -5. 最终答案必须能追溯到真实表、真实字段、查询范围、筛选/排序/聚合条件和必要的连接键。 -6. 一次性原始记录查询优先用 `+record-list` / `+record-search` 的 filter/sort；聚合分析优先用 `+data-query`；要把结果长期显示在表里，才考虑新增 `formula` / `lookup` 字段。 -7. `+data-query` 可返回聚合结果或维度字段行，但维度行按字段组合去重且不返回 `record_id`；需要逐条记录、记录定位或完整行级字段时，再用 `+record-list` / `+record-search` / `+record-get` 回查。 - ## 写入前置规则 - 优先用写入返回确认结果；返回信息不足或任务明确要求核验时，再读回。 @@ -124,9 +110,9 @@ metadata: ## 表单与视图细节 -- Base 内表单 list/get/create/update/delete 和题目管理都属于具体数据表：第一个管理命令前必须已有归属明确的真实 `table_id`；缺失或归属不明确时才用 `+table-list` 或 `+base-block-list` 定位，已有真实 ID 时直接复用。后续管理命令始终传同一 `base_token + table_id`。`+form-detail` 是分享表单入口，标识域不同，只使用 `share_token`。 +- Base 内表单 list/get/create/update/delete 和题目管理都属于具体数据表：第一个管理命令前必须已有归属明确的真实 `table_id`；缺失或归属不明确时才用 `+table-list` 或 `+base-block-list` 定位，已有真实 ID 时直接复用。后续管理命令始终传同一 `base_token + table_id`。 - 表单问题由数据表字段承载，question `id` 就是 `field_id`。创建问题前先 `+form-questions-list`；除非用户明确要求同名的独立问题，否则标题已存在时优先用 `+form-questions-update` 修改必填状态、标题或描述，不要先创建同名问题再删除旧问题。 -- `+form-questions-delete` 会删除承载问题的数据表字段。主字段问题不可删除；不要把主字段 ID 放入 `--question-ids`，需要修改时使用 `+form-questions-update`。 +- `+form-questions-delete` 用于删除非主字段问题；主字段问题使用 `+form-questions-update` 修改。 - `+form-submit` 是高风险写操作，必须带 `--yes` 确认；调用前必须先跑 `+form-detail`，读取 `questions[].type`、`required`、`filter` 和附件场景需要的 `base_token`；不要填写被 filter 隐藏的问题。 - `+form-questions-update` 是题目配置全量覆盖，不是 patch；未传字段会回落默认值，传空字符串 / `null` / 空数组会直接写入空或清空。更新前先 `+form-questions-list` 读取当前题目，把要保留的 `title` / `description` / `required` / `option_display_mode` / `visible_rule` 等字段带回请求。 - 表单附件不要写进 `fields`，放在 `--json.attachments`；提交附件时必须同时传表单所属 Base 的 `--base-token`。 @@ -152,16 +138,17 @@ metadata: | `1254015` 字段值类型不匹配 | 先 `+field-list`，再按 [lark-base-cell-value.md](references/lark-base-cell-value.md) 构造 CellValue | | `Invalid discriminator value`（字段写入缺 `type`） | 按完整提交规则读取当前字段，只改目标内容后提交；不要只补 `type` 重试 | | filter 报 `value of type array` / `Only string values` | 用 record/view 的 tuple `--filter-json`（非 `+data-query` 对象型），value 按字段 type 选标量或数组；见 [lark-base-view-set-filter.md](references/lark-base-view-set-filter.md) | -| 日期 / 人员 / 超链接字段报格式错误 | 日期用 `YYYY-MM-DD HH:mm:ss`；人员用 `[{ "id": "ou_xxx" }]`；超链接用 URL 或 markdown link 字符串 | +| 日期 / 人员 / 超链接字段报格式错误 | 日期用 `YYYY-MM-DD HH:mm`；人员用 `[{ "id": "ou_xxx" }]`；超链接用 URL 或 markdown link 字符串 | | formula / lookup 创建失败 | 先读 [formula-field-guide.md](references/formula-field-guide.md) / [lookup-field-guide.md](references/lookup-field-guide.md)，再按 guide 重建请求 | | `ignored_fields` / `READONLY` | 移除只读字段，只写存储字段 | | `1254104` | 批量超过 200，分批调用 | | `1254291` | 并发写冲突，串行写入并在批次间短暂等待 | -| `91403` | 无权限访问该 Base，按 `lark-shared` 权限流程处理，不要盲目重试 | ## 保留 Reference -- [lark-base-data-analysis-sop.md](references/lark-base-data-analysis-sop.md)：查询/统计/全局结论的选路 SOP +- [lark-base-data-analysis-sop.md](references/lark-base-data-analysis-sop.md)：所有数据表记录查询和分析的统一入口；依次选择 jq、Python 或 Cloud +- [Python 标准库](references/lark-base-data-analysis-python-stdlib.md) / [pandas](references/lark-base-data-analysis-pandas.md)：统一数据分析 SOP 选定 Python 实现后按需读取的同场景示例 +- [lark-base-data-analysis-cloud.md](references/lark-base-data-analysis-cloud.md)：统一 SOP 判定 jq 与 Python 路径均不适用时的云端查询 SOP - [lark-base-data-query-guide.md](references/lark-base-data-query-guide.md) / [lark-base-data-query.md](references/lark-base-data-query.md)：聚合查询入口 fewshot 与 DSL SSOT；`+data-query` 的 `filters` 结构是独立对象 DSL，不使用公共 tuple filter 协议 - [lark-base-cell-value.md](references/lark-base-cell-value.md)：记录 CellValue 构造 - [lark-base-field-json.md](references/lark-base-field-json.md)：字段 JSON 构造 @@ -169,7 +156,7 @@ metadata: - [lark-base-field-create.md](references/lark-base-field-create.md) / [lark-base-field-update.md](references/lark-base-field-update.md)：字段创建/更新命令级补充 - [lark-base-record-upsert.md](references/lark-base-record-upsert.md) / [lark-base-record-batch-create.md](references/lark-base-record-batch-create.md) / [lark-base-record-batch-update.md](references/lark-base-record-batch-update.md) / [lark-base-record-history-list.md](references/lark-base-record-history-list.md)：记录写入 JSON 与历史返回解释 - [lark-base-view-set-filter.md](references/lark-base-view-set-filter.md)：视图筛选 JSON -- [lark-base-filter-condition.md](references/lark-base-filter-condition.md)：视图 filter、记录 `--filter-json`、表单 `visible_rule` 的 tuple 条件结构公共协议 SSOT；不适用于 `+data-query` +- [lark-base-filter-condition.md](references/lark-base-filter-condition.md)：视图 filter、记录 `--filter-json`、表单 `visible_rule` 的 tuple 条件结构公共协议 SSOT - [lark-base-form-detail.md](references/lark-base-form-detail.md) / [lark-base-form-submit.md](references/lark-base-form-submit.md) / [lark-base-form-questions-create.md](references/lark-base-form-questions-create.md) / [lark-base-form-questions-update.md](references/lark-base-form-questions-update.md)：表单详情、提交和复杂 JSON - [lark-base-dashboard.md](references/lark-base-dashboard.md) / [dashboard-block-data-config.md](references/dashboard-block-data-config.md) / [lark-base-dashboard-block-get-data.md](references/lark-base-dashboard-block-get-data.md)：仪表盘、组件配置与图表结果协议 - [lark-base-workflow-guide.md](references/lark-base-workflow-guide.md) / [lark-base-workflow-schema.md](references/lark-base-workflow-schema.md)：workflow 入口与 steps JSON SSOT diff --git a/skills/lark-base/references/lark-base-cell-value.md b/skills/lark-base/references/lark-base-cell-value.md index 71447dc560..b1a9e44b74 100644 --- a/skills/lark-base/references/lark-base-cell-value.md +++ b/skills/lark-base/references/lark-base-cell-value.md @@ -48,25 +48,29 @@ text 字段的 `style.type` 影响单元格检查逻辑： ### 2.3 select（单选/多选） -`select` 字段用 `multiple` 区分单选和多选：`multiple=false` 时传选项名字符串，`multiple=true` 时传选项名数组。只支持写入字段中已有的选项；构造 CellValue 前先用 `+field-list` 或 `+field-search-options` 确认目标选项存在。 +`select` 字段统一传选项名称数组。`multiple=false` 时数组只能包含一个元素，`multiple=true` 时可以包含多个元素。只支持写入字段中已有的选项；构造 CellValue 前先用 `+field-list` 或 `+field-search-options` 确认目标选项存在。 ```json { - "单选": "Todo", + "单选": ["Todo"], "多选": ["后端", "高优"] } ``` +读取单元格时与写入的数据结构一致。 + ### 2.4 datetime -优先用 `YYYY-MM-DD HH:mm:ss` 字符串，这是最稳妥的写法，也和常见 API 输出更容易对齐。不要写相对时间（如“明天上午”）。 +写入可省略时区偏移量，系统会按 Base 时区解析输入字符串；优先使用 `YYYY-MM-DD HH:mm`。Base 默认按分钟展示，但底层以毫秒级精度存储时间 ```json { - "截止时间": "2026-03-24 10:00:00" + "截止时间": "2026-03-24 10:00" } ``` +读取单元格时，日期时间输出为标准 RFC3339 字符串并固定保留三位毫秒，例如 `"2026-03-24T10:00:00.000+08:00"`。 + ### 2.5 checkbox 用 JSON boolean：`true` 或 `false`，不要用 `"true"`、`"是"`、`1`。 @@ -79,7 +83,7 @@ text 字段的 `style.type` 影响单元格检查逻辑： ### 2.6 user / group_chat -用对象数组，元素至少包含 `id`。人员字段传用户 ID（如 `ou_xxx`），群字段传群 ID（如 `oc_xxx`）；单值/多值都统一使用数组。 +`user` 和 `group_chat` 字段统一传对象数组。`multiple=false` 时数组只能包含一个元素，`multiple=true` 时可以包含多个元素。每个元素至少包含 `id`；人员字段传用户 ID（如 `ou_xxx`），群字段传群 ID（如 `oc_xxx`）。 > **人员字段：不要猜 ID。** 不知道 `open_id` 时，先用 `lark-contact` 查 id：`lark-cli contact +search-user --query "<姓名/邮箱/手机号>" --as user`。 @@ -97,6 +101,8 @@ text 字段的 `style.type` 影响单元格检查逻辑： } ``` +读取单元格时仍为对象数组，每个元素为 `{id, name}`，例如 `[{"id":"ou_xxx","name":"张三"}]`。 + ### 2.7 link 用对象数组，元素包含 `id`，值为目标记录的 `record_id`。不要传记录标题；先用 `+record-list` / `+record-search` 找到目标记录 ID。 @@ -109,6 +115,8 @@ text 字段的 `style.type` 影响单元格检查逻辑： } ``` +读取单元格时与写入的数据结构一致。 + ### 2.8 location 写入对象必须使用 `{lng, lat}`，两者都是数字；`lng` 是经度，`lat` 是纬度。不需要手动传 `full_address`，平台会根据坐标解析地址。 @@ -122,10 +130,12 @@ text 字段的 `style.type` 影响单元格检查逻辑： } ``` -读取、筛选、转文本等场景使用 `full_address` 字符串；只有公式能访问坐标。如果用户只给地址文本，先获取或确认坐标后再写入；不要把仅有地址文本直接当作 location CellValue。 +读取单元格时，非空 location 为 `{lng, lat, full_address}`，三个成员均非空，`full_address` 是字符串；筛选、转文本等场景使用 `full_address`，只有公式能访问坐标。如果用户只给地址文本，先获取或确认坐标后再写入；不要把仅有地址文本直接当作 location CellValue。 ### 2.9 attachment（不作为普通 CellValue 写入） +读取单元格时，附件为数组，每个元素为 `{file_token, size, name}`，例如 `[{"file_token":"box_xxx","size":1024,"name":"report.pdf"}]`。 + - 追加附件：使用 `lark-cli base +record-upload-attachment --record-id --field-id --file `；可重复 `--file` 一次追加多个附件，不能用普通记录操作接口写附件值。 - 删除附件：使用 `lark-cli base +record-remove-attachment --record-id --field-id --file-token --yes`；可重复 `--file-token` 一次删除同一单元格里的多个附件。 - 下载附件：使用 `lark-cli base +record-download-attachment --record-id --file-token --output `；不传 `--file-token` 时下载整行所有附件，也可重复 `--file-token` 只下载指定附件。Base 附件必须用这个命令下载，用其他下载入口可能失败。 @@ -141,6 +151,8 @@ text 字段的 `style.type` 影响单元格检查逻辑： 写入只读字段通常不会更新数据；返回里可能出现 `ignored_fields`，reason 会说明 `READONLY`。看到这种返回时，不要重试同一 payload，应移除只读字段，只写存储字段。 +读取单元格时，`auto_number`、`formula`、`lookup` 为 `string|null`；`created_at`、`updated_at` 为标准 RFC3339 字符串或 `null`；`created_by`、`updated_by` 为 `array<{id, name}>`。 + ## 4. 完整示例 ```json @@ -149,7 +161,7 @@ text 字段的 `style.type` 影响单元格检查逻辑： "状态": "Todo", "标签": ["高优", "外部依赖"], "工时": 8, - "截止时间": "2026-03-24 10:00:00", + "截止时间": "2026-03-24 10:00", "已完成": false, "负责人": [{ "id": "ou_123" }], "关联任务": [{ "id": "rec_456" }], diff --git a/skills/lark-base/references/lark-base-data-analysis-cloud.md b/skills/lark-base/references/lark-base-data-analysis-cloud.md new file mode 100644 index 0000000000..5fa2ebce23 --- /dev/null +++ b/skills/lark-base/references/lark-base-data-analysis-cloud.md @@ -0,0 +1,146 @@ +# Base cloud data analysis SOP + +仅在统一数据分析 SOP 判定 jq 无法完成，并且 Python 不可用或谓词下推后的最大单表导出量仍超过 2000 条时，使用本 SOP。覆盖记录读取、筛选、排序、Top/Bottom N、聚合统计、分组聚合、多表关联和查询后写入前的目标定位。 + +本文只管查询选路和正确性边界；具体操作前先读真实结构和现状，复杂 JSON 再跳到 reference： + +- `+data-query`: entry guide [lark-base-data-query-guide.md](lark-base-data-query-guide.md), full DSL SSOT [lark-base-data-query.md](lark-base-data-query.md) +- 视图筛选: [lark-base-view-set-filter.md](lark-base-view-set-filter.md) +- 记录读取: `+record-list` / `+record-search` / `+record-get`，先确认字段 ID、字段名、分页和投影范围 + +## 0. 执行约定 + +- “最高、最低、最新、最早、Top、Bottom、总数、全部、异常、最大、最小、最多、最少、优先级最高”等全局语义，在本路径中由 Base 云端查询服务完成筛选、排序或聚合。 +- 一次性原始记录查询优先用 `+record-list` / `+record-search` 的 filter/sort；聚合分析优先用 `+data-query`。 +- `+record-search` 用于关键词检索字段的展示文本；金额、状态、日期、空值、关联等结构化条件继续用 `--filter-json` 表达。 +- 不要依赖已有视图，除非用户明确指定该视图，或你已读取并验证其 filter/sort/projection 符合当前问题。 +- 内部 ID、`record_id`、关联记录 ID、open_id 和编码字段用于连接或定位；交付输出使用用户可读的真实字段值，用户明确要求 ID 时一并展示。 +- 每次读取必须做最小投影，并包含后续解释、回查或写入需要的业务 key。 + +## 1. Intent -> Tool Path + +| 用户意图 | 首选路径 | 关键规则 | +| --- | --- | --- | +| 看几条、预览、示例 | `+record-list --limit N --field-id ...` | 保持局部语义 | +| 已知 `record_id` | `+record-get` | 直接读取 | +| 明确关键词 | `+record-search --keyword ... --search-field ... --field-id ...` | 必须显式指定 `--search-field`；可叠加 `--filter-json` | +| 按条件找原始记录 | `+record-list --filter-json ...` | `filter-json` 与视图筛选结构一致，支持文本、数字、日期、选项、人员、群组、关联等值 | +| 排序 / TopN 原始记录 | `+record-list --filter-json ... --sort-json ... --limit N` | 最高/最新用 `desc:true`，最低/最早用 `desc:false`；数组顺序表达优先级；最多 10 个排序条件 | +| 聚合 / 分组 / 分组排序 | `+data-query` | 使用 filters/dimensions/measures/sort/limit | +| 聚合后输出逐条记录 | `+data-query` 得到业务 key 或候选字段组合 -> `+record-list --filter-json` / `+record-get` 回查 | `+data-query` 维度行按字段组合去重且不返回 `record_id` | +| 多表 / 多跳关联 | 以候选数最小的事实表为驱动表，沿业务 key 或 Link 逐跳回查 | 读出 Link 单元格的 `id`（目标表 `record_id`）后，到被关联表批量 `+record-get` 展示字段 | +| 查询后写入 / 视图化 | 先用本 SOP 得到可复核的目标记录 id 集合 | 再进入记录写入或视图配置；高价值可复用查询可沉淀为持久视图 | + +## 2. Execution Patterns + +### 2.1 结构化原始记录与 TopN + +使用 `+record-list` 的 filter/sort 路径： + +1. `+field-list` 确认筛选字段、排序字段、展示字段、业务 key。 +2. 筛选使用 `--filter-json ''`。 +3. 排序用 `--sort-json`。 +4. `--field-id` 做最小投影，`--limit` 控制返回数量。 + +Example: 结构化筛选 + TopN；示例展示文本包含、数字比较和 Select 集合相交三个常用谓词： + +```bash +lark-cli base +record-list \ + --base-token \ + --table-id \ + --filter-json '{"logic":"and","conditions":[["Title","intersects","Launch plan"],["Score",">=",80],["Status","intersects",["Doing"]]]}' \ + --sort-json '[{"field":"Updated","desc":true}]' \ + --field-id Name \ + --field-id Title \ + --field-id Score \ + --limit 20 +``` + +常用 `filter-json` condition fewshot 统一见 [Base 数据表查询与分析 SOP](lark-base-data-analysis-sop.md)；完整协议见 [Base Filter 条件结构](lark-base-filter-condition.md)。 + +`--sort-json` 传排序数组，数组顺序就是优先级，`desc:true` 为降序，`desc:false` 为升序，最多 10 个排序条件。 + +### 2.2 关键词检索后叠加结构化条件 + +使用 `+record-search` 做关键词命中，结构化条件仍用 `--filter-json` 下推： + +```bash +lark-cli base +record-search \ + --base-token \ + --table-id \ + --keyword Alice \ + --search-field Name \ + --filter-json '{"logic":"and","conditions":[["Status","intersects",["Doing"]]]}' \ + --sort-json '[{"field":"Updated","desc":true}]' \ + --field-id Name \ + --field-id Status \ + --limit 20 +``` + +金额、状态、日期、空值和关联字段等结构化条件使用 `--filter-json`；`+record-search` 处理展示文本关键词。 + +### 2.3 聚合分析与 TopN + +使用 `+data-query`： + +- 让 Base 云端查询服务完成 filters、dimensions、measures、sort、pagination.limit。 +- `pagination.limit` 是 Base 云端查询服务中的结果限制，不是本地分页扫描。 +- 常用聚合 fewshot 先读 [lark-base-data-query-guide.md](lark-base-data-query-guide.md)；字段类型、日期 value、DSL shape 以 [lark-base-data-query.md](lark-base-data-query.md) 为准。 +- `+data-query` 可返回聚合结果或维度字段行；维度字段行按字段组合去重且不返回 `record_id`，不能当逐条原始记录结果使用。 +- 需要输出逐条记录、记录定位或完整行级字段时，先用 `+data-query` 得到业务 key、分组值或候选字段组合，再用 `+record-list --filter-json` / `+record-get` 回查。 + +Example: 分组计数： + +```bash +lark-cli base +data-query \ + --base-token \ + --dsl '{"datasource":{"type":"table","table":{"tableId":""}},"dimensions":[{"field_name":"Status","alias":"status"}],"measures":[{"field_name":"Status","aggregation":"count","alias":"count"}],"shaper":{"format":"flat"}}' +``` + +Example: 汇总后取 TopN；需要过滤时按 `+data-query` 的 LiteQuery DSL reference 增加 `filters`： + +```bash +lark-cli base +data-query \ + --base-token \ + --dsl '{"datasource":{"type":"table","table":{"tableId":""}},"dimensions":[{"field_name":"Owner","alias":"owner"}],"measures":[{"field_name":"Amount","aggregation":"sum","alias":"total_amount"}],"sort":[{"field_name":"total_amount","order":"desc"}],"pagination":{"limit":10},"shaper":{"format":"flat"}}' +``` + +### 2.4 视图化与复用 + +一次性查询先用 `+record-list` / `+record-search` 的 filter/sort 验证。需要用户长期打开、共享或复用时，再把同一套 filter/sort 沉淀为视图。 + +Example: 将已验证的筛选排序写入视图： + +```bash +lark-cli base +view-set-filter \ + --base-token \ + --table-id \ + --view-id \ + --json '{"logic":"and","conditions":[["Priority","intersects",["P0"]]]}' + +lark-cli base +view-set-sort \ + --base-token \ + --table-id \ + --view-id \ + --json '{"sort_config":[{"field":"Priority","desc":true}]}' +``` + +手动配置和视图配置的优先级： + +1. `--filter-json` 覆盖 `--view-id` 保存的 view filter JSON。 +2. `--sort-json` 覆盖 `--view-id` 保存的 view sort config。 +3. 没有手动 filter/sort 时，`--view-id` 使用视图自身保存的 filter/sort。 + +### 2.5 关系查询与回查 + +- Link 单元格中的元素形如 `{"id":"rec_xxx"}`；`id` 是目标表的 `record_id`，用于关系连接。 +- 先用 `+field-list` 确认 link 字段的 `link_table`、业务唯一键和展示字段。 +- 从驱动表拿到候选记录后，用 Link 元素的 `id` 到目标表 `+record-get` 批量读取记录内容。 +- 多跳关系逐跳建立 `record_id/key -> 用户可读字段` 映射，交付目标表返回的真实业务字段。 + +## 3. Range & Pagination Contract + +- `+record-list` 默认页、固定 `--limit` 和手工浏览输出都只覆盖已读取范围；模型上下文接收云端收敛后的最终小结果。 +- `has_more=true` 说明可能还有未读取数据，需要更新 offset 后继续读取，多次读取仍未读取完成时，采用其他方法完成任务需求，避免无限循环。 +- 对全局问题，只有 Base 云端查询服务已经通过 filter/sort/aggregate 收敛目标范围，或 `+data-query` 已在云端完成聚合、排序和限制时，才可以用有限返回形成结论。 +- 需要完整原始记录但云端能力无法把结果安全收敛到可返回范围时，明确说明能力边界；不要用手工分页、拆分下载或采样伪装成全局分析。 diff --git a/skills/lark-base/references/lark-base-data-analysis-pandas.md b/skills/lark-base/references/lark-base-data-analysis-pandas.md new file mode 100644 index 0000000000..a6d4fb18f1 --- /dev/null +++ b/skills/lark-base/references/lark-base-data-analysis-pandas.md @@ -0,0 +1,93 @@ +# Base NDJSON：pandas 示例 + +仅在统一数据分析 SOP 已选择 pandas 后读取。本页不重复 Base 的粒度与关系规则，只展示对应实现。 + +示例假设 `records.ndjson` 包含 `record_id`、`日期`、`状态`、`金额`、`负责人`、`标签`、`关联客户`；`customers.ndjson` 包含 `record_id`、`客户名称`。多值列使用统一数据分析 SOP 定义的数组结构。 + +## 加载与日期解析 + +```python +import pandas as pd + +records = pd.read_json("records.ndjson", lines=True) +raw_dates = records["日期"].astype("string") +records["日期_local"] = pd.to_datetime( + raw_dates.str.slice(0, 10), format="%Y-%m-%d", errors="coerce" +) +records["日期_instant"] = pd.to_datetime( + raw_dates, format="ISO8601", utc=True, errors="coerce" +) +``` + +按来源 Base 的日、周、月分组使用 `日期_local`；计算真实时长、排序或跨时区比较使用 `日期_instant`。实际任务只需构造所需的一列。 + +## 集合谓词：保持 record 粒度 + +筛选“状态”包含“进行中”的记录，并在 record 粒度汇总： + +```python +active = records[records["状态"].map(lambda values: "进行中" in values)] +summary = { + "records_count": len(active), + "amount_sum": active["金额"].sum(min_count=1), +} +``` + +## 单数组展开：切换到人员粒度 + +```python +owners = records[["record_id", "负责人"]].explode("负责人", ignore_index=True) +owners = owners[owners["负责人"].notna()].assign( + user_id=lambda df: df["负责人"].map(lambda user: user["id"]), + user_name=lambda df: df["负责人"].map(lambda user: user["name"]), +) +by_owner = ( + owners.groupby(["user_id", "user_name"], as_index=False) + .agg(records_count=("record_id", "nunique")) + .sort_values("records_count", ascending=False) +) +``` + +## Link JOIN：先建立边表 + +```python +edges = ( + records[["record_id", "关联客户"]] + .rename(columns={"record_id": "source_record_id"}) + .explode("关联客户", ignore_index=True) +) +edges = edges[edges["关联客户"].notna()].assign( + target_record_id=lambda df: df["关联客户"].map(lambda link: link["id"]) +)[["source_record_id", "target_record_id"]] + +customers = pd.read_json("customers.ndjson", lines=True).rename( + columns={"record_id": "target_record_id"} +) +joined = edges.merge( + customers[["target_record_id", "客户名称"]], + on="target_record_id", + how="left", +) +``` + +## 多数组共现：显式生成行内笛卡尔积 + +连续两次 `explode` 表示同一 source record 内的 `负责人 × 标签`： + +```python +pairs = ( + records[["record_id", "负责人", "标签"]] + .explode("负责人", ignore_index=True) + .explode("标签", ignore_index=True) + .dropna(subset=["负责人", "标签"]) + .assign( + user_id=lambda df: df["负责人"].map(lambda user: user["id"]), + user_name=lambda df: df["负责人"].map(lambda user: user["name"]), + ) +) +cooccurrence = ( + pairs.groupby(["user_id", "user_name", "标签"], as_index=False) + .agg(records_count=("record_id", "nunique")) + .sort_values("records_count", ascending=False) +) +``` diff --git a/skills/lark-base/references/lark-base-data-analysis-python-stdlib.md b/skills/lark-base/references/lark-base-data-analysis-python-stdlib.md new file mode 100644 index 0000000000..559f91264a --- /dev/null +++ b/skills/lark-base/references/lark-base-data-analysis-python-stdlib.md @@ -0,0 +1,120 @@ +# Base NDJSON：Python 标准库示例 + +仅在统一数据分析 SOP 已选择 Python 标准库后读取。本页不重复 Base 的粒度与关系规则，只展示对应实现。 + +示例假设 `records.ndjson` 包含 `record_id`、`日期`、`状态`、`金额`、`负责人`、`标签`、`关联客户`；`customers.ndjson` 包含 `record_id`、`客户名称`。多值列使用统一数据分析 SOP 定义的数组结构。 + +## 加载与日期解析 + +NDJSON 每行是一条独立 JSON record；按行读取即可，不要先把文件整体载入字符串。 + +```python +import json +from datetime import date, datetime + + +def read_ndjson(path): + with open(path, encoding="utf-8") as stream: + for line in stream: + if line.strip(): + yield json.loads(line) + + +records = list(read_ndjson("records.ndjson")) +for record in records: + raw_date = record["日期"] + record["日期_local"] = date.fromisoformat(raw_date[:10]) if raw_date else None + record["日期_instant"] = datetime.fromisoformat(raw_date) if raw_date else None +``` + +按来源 Base 的日、周、月分组使用 `日期_local`；计算真实时长、排序或跨时区比较使用 `日期_instant`。实际任务只需构造所需的一项。 + +## 集合谓词：保持 record 粒度 + +筛选“状态”包含“进行中”的记录，并在 record 粒度汇总： + +```python +active = [record for record in records if "进行中" in record["状态"]] +amounts = [record["金额"] for record in active if record["金额"] is not None] +summary = { + "records_count": len(active), + "amount_sum": sum(amounts) if amounts else None, +} +``` + +## 单数组展开：切换到人员粒度 + +用嵌套循环表达 lateral expansion；按人员 `id` 聚合，`name` 只用于展示。 + +```python +from collections import defaultdict + +record_ids_by_owner = defaultdict(set) +owner_names = {} +for record in records: + for owner in record["负责人"]: + record_ids_by_owner[owner["id"]].add(record["record_id"]) + owner_names[owner["id"]] = owner["name"] + +by_owner = sorted( + ( + { + "user_id": user_id, + "user_name": owner_names[user_id], + "records_count": len(record_ids), + } + for user_id, record_ids in record_ids_by_owner.items() + ), + key=lambda row: (-row["records_count"], row["user_id"]), +) +``` + +## Link JOIN：先建立目标表索引 + +目标表的 `record_id` 是唯一主键，可直接建立哈希索引；Link 的 `id` 用于索引查找。 + +```python +customers = { + customer["record_id"]: customer + for customer in read_ndjson("customers.ndjson") +} + +joined = [] +for record in records: + for link in record["关联客户"]: + customer = customers.get(link["id"]) + joined.append( + { + "source_record_id": record["record_id"], + "target_record_id": link["id"], + "客户名称": customer["客户名称"] if customer else None, + } + ) +``` + +## 多数组共现：显式生成行内笛卡尔积 + +两层嵌套循环表示同一 source record 内的 `负责人 × 标签`： + +```python +record_ids_by_pair = defaultdict(set) +owner_names = {} +for record in records: + for owner in record["负责人"]: + owner_names[owner["id"]] = owner["name"] + for tag in record["标签"]: + record_ids_by_pair[(owner["id"], tag)].add(record["record_id"]) + +cooccurrence = sorted( + ( + { + "user_id": user_id, + "user_name": owner_names[user_id], + "标签": tag, + "records_count": len(record_ids), + } + for (user_id, tag), record_ids in record_ids_by_pair.items() + ), + key=lambda row: (-row["records_count"], row["user_id"], row["标签"]), +) +``` diff --git a/skills/lark-base/references/lark-base-data-analysis-sop.md b/skills/lark-base/references/lark-base-data-analysis-sop.md index 62c1b2fdee..44ce663fca 100644 --- a/skills/lark-base/references/lark-base-data-analysis-sop.md +++ b/skills/lark-base/references/lark-base-data-analysis-sop.md @@ -1,210 +1,220 @@ -# Base data analysis SOP - -Base 数据查询与分析任务的执行契约。覆盖记录读取、筛选、排序、Top/Bottom N、聚合统计、分组聚合、多表关联、临时分析和查询后写入前的目标定位。 +# Base 数据表查询与分析 SOP + +所有数据表记录查询和分析任务先读本 SOP，包括记录预览、`+record-get`、`+record-list`、`+record-search`、`+data-query`、筛选、排序、去重、统计、聚合、TopN、多值计算、Link 或多表关联、复杂行级计算、全局结论和查询后写入。先区分需要 LLM 理解原文的语义分析与可程序化计算的确定性分析，再按任务所需数据规模与计算复杂度选择对应路径。 + +## 分流决策 + +1. 明确所有需要参与分析的表及其 `records_count`。 +2. 如果结论必须依赖 LLM 理解原始内容，例如开放文本打标、情绪或意图识别、主题归纳、语义分类、相似性判断或实体消歧，进入下文“LLM 语义分析”路径。 +3. 对于其余确定性查询，任一分析表超过 2000 行时，先从任务意图中为所有大表提取可在单表内独立执行的谓词，例如日期范围、状态和关键词，再按下文将谓词逐表下推，并用 `--field-id '<一个简单标量字段>' --limit 2000 --output .ndjson --minimal-stdout` 探测。目标是每张表都达到 `has_more=false`；任一表无法压缩到 2000 行以内时，转 [lark-base-data-analysis-cloud.md](lark-base-data-analysis-cloud.md) 用云端的数据分析能力。 +4. 所有分析表都不超过 2000 行后：若只有一张表且短 jq 可清晰完成筛选、计数、简单分组/聚合/排序、TopN 可以使用 jq。 +5. 其余确定性任务比如多表、日历计算和复杂数据分析，在 Python 可用时使用 Python，否则用云端的数据分析能力。 + +## 执行与交付 + +分析输入默认采用 `--output x.ndjson`；`--format json` 和 Markdown 适用于向用户即时展示的小结果。 + +缩小大表记录范围时，展示文本关键词用 `+record-search`，日期、状态、数字、空值、选项、人员和关联等结构化条件用 `+record-list --filter-json`。 + +### 单表谓词下推常用 example + +`+record-list` / `+record-search` 的 `--filter-json ''` 支持使用 tuple condition 下推单表谓词。以下示例用注释说明各条件的含义；实际传参时删除注释并使用标准 JSON： + +```jsonc +{ + "logic": "and", // 所有 conditions 同时成立；任意一个成立时使用 "or" + "conditions": [ + ["标题", "==", "Launch plan"], // 文本全等 + ["标题", "intersects", "urgent"], // 文本包含目标片段 + ["金额", ">=", 100], // 数字比较；支持 ==、!=、>、>=、<、<= + ["状态", "intersects", ["进行中", "暂停"]], // Select 集合相交：包含“进行中”或“暂停”任意一个选项 + ["状态", "disjoint", ["已终止"]], // Select 集合无交集 + ["已完成", "==", true], // Checkbox + ["负责人", "intersects", [{"id": "ou_xxx"}]], // 负责人包含某个人；intersects 表示包含数组中任意一个人员 + ["关联项目", "intersects", [{"id": "rec_xxx"}]], // 关联项目包含某个 record_id；intersects 表示包含数组中任意一条关联 + ["备注", "non_empty"], // 非空判断；标量 null 和多值空数组都是空 + ["业务日期", "==", "ExactDate(2026-08-07)"], // 具体一天：按 Base 时区匹配 2026-08-07 当天 + ["发生时间", ">", "ExactDate(2024-01-31 23:59:59)"], // 2024 年 2 月范围下界：闰年 2 月包含 29 日 + ["发生时间", "<", "ExactDate(2024-03-01 00:00:00)"] // 2024 年 2 月范围上界：小于 3 月 1 日零点 + ] +} +``` -本文只管查询选路和正确性边界；具体操作前先读真实结构和现状，复杂 JSON 再跳到 reference： +全表分析的常规资源链路是 `+table-list` 确认目标表与规模，对所有参与分析的表并发执行 `+field-list` 读取所需 schema，再用 `+record-list` 导出记录；已有可信的 `table_id` 时可直接并发读取各表 `+field-list`。`+view-get` 可按需读取，作为用户持久化访问习惯的可选参考；其中的 filter、sort 与字段范围可辅助理解用户常用的查询范围和排序偏好，并结合当前任务确定最终口径。 -- `+data-query`: entry guide [lark-base-data-query-guide.md](lark-base-data-query-guide.md), full DSL SSOT [lark-base-data-query.md](lark-base-data-query.md) -- 视图筛选: [lark-base-view-set-filter.md](lark-base-view-set-filter.md) -- 记录读取: `+record-list` / `+record-search` / `+record-get`，先确认字段 ID、字段名、分页和投影范围 +1. 每次读取使用任务所需的最小投影，并包含 JOIN、解释、回查或写入需要的业务 key。 +2. 全局结论以 `has_more=false` 的完整导出或 Cloud 聚合结果为依据；`has_more=true` 表示当前结果仅覆盖已读取范围。 +3. 确定性分析选定一个分析引擎直接读取 NDJSON；模型上下文仅接收预览或最终小结果。 +4. 任务涉及业务键、展开、JOIN 或金额分摊时，明确目标粒度并检查与口径直接相关的空值、重复或总量守恒。 +5. 最终结果保留真实表、查询范围和计算口径，展示用户可读字段；内部 ID 用于连接或定位。 -## 0. Hard Rules +`+table-list` / `+base-block-list` 返回的 `records_count` 表示整表行数；manifest 的 `records_count` 表示本次查询实际导出的行数。 -- 全局问题不能用默认 `+record-list --limit N` 片面地回答。 -- `jq` / shell / 本地代码是在个人电脑或当前运行环境中处理已返回数据，只适合小范围结果；超过 200 行默认不推荐本地统计、排序或求极值，应改用 Base 云端查询服务的 filter/sort/aggregate。 -- “最高、最低、最新、最早、Top、Bottom、总数、全部、异常、最大、最小、最多、最少、优先级最高”等全局语义，必须在 Base 云端查询服务中完成筛选、排序或聚合。 -- 一次性原始记录查询优先用 `+record-list` / `+record-search` 的 filter/sort；聚合分析优先用 `+data-query`。 -- `+record-search` 用于关键词检索字段的展示文本；金额、状态、日期、空值、关联等结构化条件继续用 `--filter-json` 表达。 -- 不要依赖已有视图，除非用户明确指定该视图，或你已读取并验证其 filter/sort/projection 符合当前问题。 -- 交付输出必须使用用户可读的真实字段值；内部 ID、`record_id`、关联记录 ID、open_id、编码字段只可作为连接键或定位键，不能替代最终输出，除非用户明确要求输出这些键值。 -- 每次读取必须做最小投影，并包含后续解释、回查或写入需要的业务 key。 +## 复用本轮 NDJSON -## 1. Intent -> Tool Path +Agent 上下文曾下载过当前表的 NDJSON 时，按以下规则判断是否复用： -| 用户意图 | 首选路径 | 关键规则 | -| --- | --- | --- | -| 看几条、预览、示例 | `+record-list --limit N --field-id ...` | 保持局部语义；不要推广为全局结论 | -| 已知 `record_id` | `+record-get` | 直接读取；不要 search/list 反查 | -| 明确关键词 | `+record-search --keyword ... --search-field ... --field-id ...` | 必须显式指定 `--search-field`；可叠加 `--filter-json` | -| 按条件找原始记录 | `+record-list --filter-json ...` | `filter-json` 与视图筛选结构一致，支持文本、数字、日期、选项、人员、群组、关联等值 | -| 排序 / TopN 原始记录 | `+record-list --filter-json ... --sort-json ... --limit N` | 最高/最新用 `desc:true`，最低/最早用 `desc:false`；数组顺序表达优先级；最多 10 个排序条件 | -| 聚合 / 分组 / 分组排序 | `+data-query` | 使用 filters/dimensions/measures/sort/limit | -| 聚合后输出逐条记录 | `+data-query` 得到业务 key 或候选字段组合 -> `+record-list --filter-json` / `+record-get` 回查 | `+data-query` 维度行按字段组合去重且不返回 `record_id` | -| 多表 / 多跳关联 | 以候选数最小的事实表为驱动表，沿业务 key 或 link `record_id` 逐跳回查 | 读出 link 单元格里的关联 `record_id` 后，到被关联表批量 `+record-get` 展示字段 | -| 查询后写入 / 视图化 | 先用本 SOP 得到可复核的目标记录 id 集合 | 再进入记录写入或视图配置；高价值可复用查询可沉淀为持久视图 | +1. 短时间内继续分析或表中数据低频变化时，谓词下推口径一致且已有列覆盖计算需求即可优先复用。 +2. 间隔较长或表中数据高频变化时，批量提取 manifests 的 `base_token/table_id/rev`，并发执行 `+table-list` 校验最新 `rev`；版本一致且谓词口径未变时复用，否则重新导出对应表。 -## 2. Execution Patterns +> 例：本轮已按“日期在 2026 年”导出 `orders.ndjson`，用户继续要求按负责人聚合；谓词和所需列未变，直接复用。若间隔较长或该表频繁写入，manifest `rev=42` 与 `+table-list` 最新 `rev` 相同则复用，最新 `rev=43` 则重新导出。 -### 2.1 结构化原始记录与 TopN +## LLM 语义分析 -使用 `+record-list` 的 filter/sort 路径： +先用任务中明确且不改变分析口径的确定性条件缩小数据范围；只有剩余判断必须依赖语义理解时，才将必要原文加载到模型上下文。 -1. `+field-list` 确认筛选字段、排序字段、展示字段、业务 key。 -2. 筛选只用 `--filter-json` 或 `--filter-json @file`。 -3. 排序用 `--sort-json`。 -4. `--field-id` 做最小投影，`--limit` 控制返回数量。 +开放文本打标、情绪或意图识别、主题归纳、语义分类、相似性判断和实体消歧等任务必须理解原文，最终判断由当前 LLM 在本地上下文中逐条完成。代码只用于确定性范围筛选、分批、结果持久化和最终汇总；除非用户明确要求规则法，不用关键词命中、词频、正则或规则打分替代语义判断。 -Example: string/number 条件 + TopN： +1. 先把日期、状态、来源等不改变任务语义的确定性范围条件下推到 Base，只导出 `record_id`、判断所需原文和最终解释所需的最小字段集。 +2. 在读取正文前，先看 manifest 的 `record_file_size_bytes`；结合 `records_count` 以及所选字符串列的 `null_count`、`max_length` 判断正文相对当前上下文的规模，拿不准时先读取前 3 行再决定读取范围。 +3. 文件较小且上下文充足时，将必要记录读入上下文并直接完成语义分析；文件较大但任务仍必须理解全部原文时，先向用户说明原因和预计耗时，在确认后按文本体量分批处理。各批沿用同一判断口径，将 `record_id`、结构化判断和必要依据持续写入本地 artifact，最后统一汇总。 -```bash -lark-cli base +record-list \ - --base-token \ - --table-id \ - --filter-json '{"logic":"and","conditions":[["Title","==","Launch plan"],["Score",">=",80]]}' \ - --sort-json '[{"field":"Updated","desc":true}]' \ - --field-id Name \ - --field-id Title \ - --field-id Score \ - --limit 20 -``` +## Manifest -Example: 复杂筛选从文件读取： +`--output .ndjson` 生成 `.ndjson` 与 `.manifest.json`；记录写入 NDJSON，stdout 返回 manifest，`--minimal-stdout` 只保留文件位置、文件字节数、`records_count` 和 `has_more`。 -```bash -lark-cli base +record-list \ - --base-token \ - --table-id \ - --filter-json @filter.json \ - --sort-json '[{"field":"Priority","desc":true}]' \ - --field-id Name \ - --field-id Tags \ - --limit 50 -``` - -`filter-json` 与视图筛选结构一致。下面只列常用 fewshot；字段类型、operator、value 形状拿不准，或需要人员、群组、关联、空值、地理位置、formula / lookup 等完整筛选时，先读 [lark-base-view-set-filter.md](lark-base-view-set-filter.md)，再把同样的 filter JSON 传给 `--filter-json`。 - -文本 `==`：字段值等于目标文本。 -```json -{"logic":"and","conditions":[["Title","==","Launch plan"]]} -``` - -文本包含 / like：文本字段包含目标片段；operator 写 `intersects`。 -```json -{"logic":"and","conditions":[["Title","intersects","urgent"]]} -``` - -数字 `==`：字段值等于目标数字。 -```json -{"logic":"and","conditions":[["Score","==",95]]} -``` +分析 artifact 尽量使用相对路径输出到当前工作目录，例如 `--output ./records.ndjson`。 -日期 `==`：字段值等于目标日期；datetime / created_at / updated_at 用 `ExactDate(...)`。 ```json -{"logic":"and","conditions":[["Due Date","==","ExactDate(2026-06-02)"]]} +{ + "record_file": "/path/records.ndjson", + "record_file_size_bytes": 18432, + "manifest_file": "/path/records.manifest.json", + "records_count": 137, + "has_more": false, + "columns": { + "record_id": {"physical_type": "string", "stats": {"max_length": 15}}, + "状态": { + "field_id": "fld_status", + "field_type": "select", + "physical_type": "array", + "stats": {"empty_count": 3, "max_length": 2, "avg_length": 1.1}, + "example": ["进行中"] + } + } +} ``` -选项 `==`：字段值匹配单个选项；选项值使用选项名数组，单个选项也写数组。 -```json -{"logic":"and","conditions":[["Priority","==",["P0"]]]} -``` - -选项 `intersects`：字段值与给定选项集合有交集，常用于多选或“命中任一选项”。 -```json -{"logic":"and","conditions":[["Tags","intersects",["P0","Blocked"]]]} -``` - -`--sort-json` 传排序数组，数组顺序就是优先级，`desc:true` 为降序，`desc:false` 为升序，最多 10 个排序条件。 +- manifest `columns` 是 NDJSON 物理 schema 的权威来源，包含 `field_id`、`field_type`、`physical_type`、`stats` 以及可选的真实 example 或 hint；它不替代完整 Base field schema，选项配置、数字格式、Link 目标表或 formula/lookup 定义影响任务时读取 `+field-list`。全空列按 hint 跳过，任务必须使用时显式 cast。 +- `stats` 只统计本次导出的 records；`null_count` 只计 JSON `null`，字符串长度按 Unicode 字符计数，数字 `avg` 排除 null，多值 `avg_length` 按全部 records（含 `[]`）计算。 + +| 列类别 | `stats` | +| --- | --- | +| 普通字符串 | `null_count, max_length` | +| 数字 | `null_count, min, max, avg` | +| 日期 | `null_count, min, max` | +| checkbox | `true_count` | +| Location | `null_count` | +| 多值列 | `empty_count, max_length, avg_length` | +| 系统 `record_id` | `max_length` | + +- stdout 的 `records_count` 和 `has_more` 描述本次导出；确认后无需在分析代码中重读 manifest 或重新统计 NDJSON 行数。 +- `record_file_size_bytes` 是 NDJSON artifact 的实际字节数，用于选择一次读取、预览或分批方式；确定性计算由 jq/Python 直接读取文件。 +- manifest 的 `rev` 是导出首个响应页返回的 table revision；与 `+table-list` 返回的最新 `rev` 比较，可判断本轮 NDJSON 是否仍对应当前表版本。 +- `query_context` 保存导出查询范围；复用本轮 NDJSON 时结合原查询上下文确认谓词下推口径保持一致。 +- 仅在需要 `columns`、example、hint 或执行 artifact 复用判断时读取 `manifest_file`；满足复用条件后直接继续分析现有 NDJSON。 +- `ignored_fields` 和 `record_not_found` 仅在 stdout 返回时关注。 + +## 数据库专家快速心智模型 + +- Base table 是面向协作的反范式宽表；本地分析将每个导出表作为关系输入，不假设数据库级约束。 +- 每行是一条 record；系统 `record_id` 是表内真正的主键，由 Base 系统生成并维护，契约保证 `NOT NULL` 和 `UNIQUE`，分析代码无需再次检查空值或唯一性，也不可把它作为普通字段更新。Base 的“主字段”只是主要展示字段，不是主键。 +- NDJSON 业务列一律使用字段 `name` 作为 key，不使用 `field_id`；字段重命名会改变 key，对应的 `field_id` 仅记录在 manifest 列元数据中。 +- 除 `record_id` 外，不假设任何列满足 `NOT NULL`、`UNIQUE` 或业务键约束；仅当某列实际作为业务键参与关联或去重时处理空值和重复值。 +- checkbox 在 NDJSON 中始终为 `true` 或 `false`，上游空值会在导出时规范化为 `false`；其他标量列可空并使用 `null`。多值列始终非空，没有元素时用 `[]`；这些是序列化契约，不是业务约束。 +- NDJSON 的读取结构以 manifest `physical_type` 和下表为准，不等同于写记录时的 CellValue；`lark-base-cell-value.md` 在读写形态不一致的类型下提供对照说明。formula 和 lookup 在当前 NDJSON 中统一为字符串，不保留计算结果的原始类型。 +- 将 `physical_type` 和上述 CellValue 结构视为输入契约；一次性分析代码直接读取，不再逐格验证 `record_id`、数组或 struct 的运行时形状。 +- 未显式指定 sort 时不保证行顺序。 + +### Physical type 快速参考 + +| `field_type` | `physical_type` | 示例与语义 | +| --- | --- | --- | +| 系统 `record_id` | `string` | `"rec_xxx"`；系统主键 | +| `text`、`formula`、`lookup`、`auto_number`、`not_support` | `string|null` | `"进行中"`；formula、lookup 不保留结果的原始类型 | +| `datetime`、`created_at`、`updated_at` | `string|null` | `"2026-08-05T10:30:00.000+08:00"`；RFC3339，固定三位毫秒 | +| `number` | `number|null` | `12.5`；JSON 整数和小数均为 number | +| `checkbox` | `boolean` | `true`；上游空值已规范化为 `false` | +| `select` | `array` | `["进行中", "高优"]`；单选、多选读取均为名称数组 | +| `location` | `struct|null` | `{"lng":116.39,"lat":39.90,"full_address":"北京市"}`；非空 Location 的三个成员均非空 | +| `user`、`group_chat`、`created_by`、`updated_by` | `array>` | `[{"id":"ou_xxx","name":"张三"}]` | +| `link` | `array>` | `[{"id":"rec_xxx"}]`；schema 的 `table_id` 指定目标表，`id` 是目标 `record_id` | +| `attachment` | `array>` | `[{"file_token":"box_xxx","size":1024,"name":"report.pdf"}]` | -### 2.2 关键词检索后叠加结构化条件 +### 日期字段读取 -使用 `+record-search` 做关键词命中，结构化条件仍用 `--filter-json` 下推： +日期字段以带 offset 的 RFC3339 字符串序列化，并有两种分析语义： -```bash -lark-cli base +record-search \ - --base-token \ - --table-id \ - --keyword Alice \ - --search-field Name \ - --filter-json '{"logic":"and","conditions":[["Status","!=","Done"]]}' \ - --sort-json '[{"field":"Updated","desc":true}]' \ - --field-id Name \ - --field-id Status \ - --limit 20 -``` +- **instant semantics**：计算真实时长、先后顺序或跨时区比较时，解析完整 RFC3339 值，以其表示的绝对时刻计算。 +- **local-calendar semantics**：按来源 Base 的日、周、月等本地日历分组时，使用序列化值中的本地日期，不先转 UTC，也不按 manifest `timezone` 重复换算。 -不要把 `+record-search` 当成金额、状态、日期、空值、关联字段的结构化筛选入口；这些条件继续写成 `--filter-json`。 +例如，`2026-03-20T23:30:00.000-05:00` 与 `2026-03-21T12:30:00.000+08:00` 表示同一时刻；前者若是来源 Base 的值，本地日报归入 3 月 20 日，而时长或排序计算应把它解析为绝对时刻。只构造任务实际需要的日期表示，并在分析引擎中使用具备 datetime 功能的列。 -### 2.3 聚合分析与 TopN +## 读取与关系建模 -使用 `+data-query`： +仅在 SOP 已选择 Python 路径后，按实际实现方式只读一份示例： -- 让 Base 云端查询服务完成 filters、dimensions、measures、sort、pagination.limit。 -- `pagination.limit` 是 Base 云端查询服务中的结果限制，不是本地分页扫描。 -- 常用聚合 fewshot 先读 [lark-base-data-query-guide.md](lark-base-data-query-guide.md)；字段类型、日期 value、DSL shape 以 [lark-base-data-query.md](lark-base-data-query.md) 为准。 -- `+data-query` 可返回聚合结果或维度字段行；维度字段行按字段组合去重且不返回 `record_id`，不能当逐条原始记录结果使用。 -- 需要输出逐条记录、记录定位或完整行级字段时，先用 `+data-query` 得到业务 key、分组值或候选字段组合，再用 `+record-list --filter-json` / `+record-get` 回查。 +- [Python 标准库示例](lark-base-data-analysis-python-stdlib.md) +- [pandas 示例](lark-base-data-analysis-pandas.md) -Example: 分组计数： +两份示例使用相同的五类场景：加载与日期解析、集合谓词、单数组展开、Link JOIN、多数组共现。场景语义和粒度规则以本 SOP 为准，示例只提供对应实现的最短代码。 -```bash -lark-cli base +data-query \ - --base-token \ - --dsl '{"datasource":{"type":"table","table":{"tableId":""}},"dimensions":[{"field_name":"Status","alias":"status"}],"measures":[{"field_name":"Status","aggregation":"count","alias":"count"}],"shaper":{"format":"flat"}}' -``` +标准库足以清晰表达任务时直接使用；DataFrame 能明显简化计算时再选 pandas。已选择 pandas 但环境未安装时，网络可用且存在 `uv` 或 `pip` 才按需安装，优先使用 `uv run --no-project --with pandas python analyze.py`。 -Example: 过滤后汇总并取 TopN： +将 Base 反范式宽表映射为关系模型时，可将标量列视为 record attributes，将多值列视为以 `record_id` 为关联键的 nested relation，将 Link 视为跨表 adjacency list。多值列通过 lateral `explode` / `UNNEST` 切换粒度；Link 规范化为 bridge relation 后再 `merge` / `join` / `JOIN`；同类来源表先投影到 conformed fact schema，再用 `concat` / `UNION ALL` 纵向合并。 -```bash -lark-cli base +data-query \ - --base-token \ - --dsl '{"datasource":{"type":"table","table":{"tableId":""}},"dimensions":[{"field_name":"Owner","alias":"owner"}],"measures":[{"field_name":"Amount","aggregation":"sum","alias":"total_amount"}],"filters":{"type":1,"conjunction":"and","conditions":[{"field_name":"Status","operator":"is","value":["Done"]}]},"sort":[{"field_name":"total_amount","order":"desc"}],"pagination":{"limit":10},"shaper":{"format":"flat"}}' -``` +## 常见分析模式 -### 2.4 视图化与复用 +### 单表简单筛选与统计：jq -一次性查询先用 `+record-list` / `+record-search` 的 filter/sort 验证。需要用户长期打开、共享或复用时，再把同一套 filter/sort 沉淀为视图。 +NDJSON 每行是一条 record。单表短筛选、计数和简单聚合可直接用 jq；下面筛选“状态”包含“进行中”的记录，并统计记录数和金额合计： -Example: 将已验证的筛选排序写入视图： +默认导出后使用本地 `jq -s`，同一 artifact 可反复查询而无需重新下载。表达式很短且只执行一次，或本地 jq 不可用时，可改用 `--jq-records ''` 等价 `js -s '' records.ndjson`；注意：通用的 `--jq` 只处理 stdout 里的内容，`--jq-records` 才能处理 NDJSON 文件内容。 ```bash -lark-cli base +view-set-filter \ - --base-token \ - --table-id \ - --view-id \ - --json @filter.json - -lark-cli base +view-set-sort \ +lark-cli base +record-list \ --base-token \ --table-id \ - --view-id \ - --json '{"sort_config":[{"field":"Priority","desc":true}]}' + --field-id 状态 \ + --field-id 金额 \ + --limit 2000 \ + --output records.ndjson \ + --minimal-stdout && +jq -s ' + map(select((.["状态"] | index("进行中")) != null)) as $records + | ($records | map(.["金额"] | select(. != null))) as $amounts + | { + records_count: ($records | length), + amount_sum: ( + if ($amounts | length) > 0 then ($amounts | add) else null end + ) + } +' records.ndjson ``` -手动配置和视图配置的优先级： - -1. `--filter-json` 覆盖 `--view-id` 保存的 view filter JSON。 -2. `--sort-json` 覆盖 `--view-id` 保存的 view sort config。 -3. 没有手动 filter/sort 时，`--view-id` 使用视图自身保存的 filter/sort。 +### 多值列：nested relation 与目标粒度 -### 2.5 关系查询与回查 +Base 的反范式宽表会把零到多个 Select、人员、群组、Link 或附件元素嵌入一条 source record。多值单元格默认按无重复、无序集合建模：元素顺序不承担稳定业务语义，同一 source record 内可将元素视为唯一，因此其元素数等于去重元素数；跨 source record 出现的同一元素仍是不同事实或关系边。分析时将数组视为以 `record_id` 为 correlation key 的 nested relation，并先确定 target grain： -- link 单元格通常是关联表 `record_id` 数组，不是用户可读内容，只是连接键。 -- 先用 `+field-list` 确认 link 字段的 `link_table`、业务唯一键和展示字段。 -- 从驱动表拿到候选记录后，用关联 `record_id` 到关联表 `+record-get` 批量读取记录内容。 -- 多跳关系逐跳建立 `record_id/key -> 用户可读字段` 映射；最终用户可读的信息。 +- **record grain**：包含、交集、子集和元素数量等问题直接使用集合谓词，不做 expansion。 +- **record-element grain**：通过 lateral `explode` / `UNNEST` 规范化为 `(source_record_id, element)` bridge relation。inner expansion 会丢弃空数组来源，outer expansion 会保留来源 record；回到 record 口径时按 `source_record_id` 聚合或去重。 +- **entity grain**：两侧分别规范化为 bridge relation，再按稳定 element key JOIN。人员和群组以 `id` 连接、以 `name` 展示；Select 以名称作为元素键，仅当字段共享同一业务值域时才可连接。 -禁止： +使用列 `stats` 中的 `empty_count`、`avg_length` 和 `max_length` 做 expansion cardinality 与数据倾斜预估：单数组 inner expansion 的估算行数为 `records_count × avg_length`，outer expansion 还需加上 `empty_count`；结合 `max_length` 识别极端 fan-out 或 hot record。任务确实需要元素粒度且估算规模可控时，可以直接展开。 -- 把 link `record_id` 当最终输出。 -- 用 `+record-search` 搜 link `record_id`。 -- 基于 ID、自增编号、link 值做语义猜测；禁止依赖字段先验、样本记忆补全交付输出。 +#### 多数组、fan-out 与 row-local Cartesian product -## 3. Range & Pagination Contract +同一 source record 中的独立数组默认建立为彼此独立的 lateral pipeline，分别展开并聚合回 target grain 后再连接，避免 many-to-many fan-out 和重复计量。只有问题明确要求分析元素组合或共现时，才同时展开形成 row-local Cartesian product。 -- `+record-list` 默认页、固定 `--limit`、本地 `jq`、shell 管道、手工浏览输出，都只覆盖已读取范围；超过 200 行不要把本地处理当作推荐路径。 -- `has_more=true`、存在下一页 offset/page token、或返回行数等于 page size，都表示可能还有未读取数据。 -- 对全局问题，只有 Base 云端查询服务已经通过 filter/sort/aggregate 收敛目标范围，或 `+data-query` 已在云端完成聚合、排序和限制时，才可以用有限返回形成结论。 -- 必须全量导出时，按 `+record-list` 分页语义串行翻页；不要并发调用 `+record-list`。 +两个数组同时展开的准确 cardinality 为 `Σᵢ(|Aᵢ| × |Bᵢ|)`；可用 `records_count × avg_length_a × avg_length_b` 估算执行规模，并结合两列的 `max_length` 判断极端 fan-out。平均长度乘积不反映列间相关性，只用于成本估算。Base schema 不提供不同多值列之间的 positional contract；仅当额外业务契约明确声明位置对应语义时，才按 ordinality ZIP。 -## 4. Final Answer Check +### Link：跨表 adjacency list -形成交付输出前必须能确认： +- Link 字段的完整 schema 以 `+field-list` 为准，其中 `table_id` 声明唯一目标 table；NDJSON 的 `[{"id":"rec_xxx"}]` 表示指向该表目标 `record_id` 的零到多条有向边。以 `table_id` 确定目标表，缺少可信 schema 时先补充 `+field-list`。 +- 将 Link 规范化为 `(source_record_id, target_record_id)` edge/bridge relation，再按 `target_record_id = 目标表.record_id` 执行外键式 JOIN。需要反向遍历时复用同一 edge relation 反向分组或连接；NDJSON 不隐含自动反向关系。 +- 多跳 Link 通过逐跳组合 edge relation 完成 traversal，并始终在各自 record-id domain 内连接。最终展示目标表的用户可读 attributes；已有 Link 时使用 Link edge relation，其他关联使用经过验证的 business key。 -- 问题范围是局部样例、单点定位、全局原始记录、聚合分析、多表关联，还是查询后写入。 -- 筛选、排序、聚合是否发生在 Base 云端查询服务中，而不是本地 `jq` / shell 中。 -- 如果使用 `jq` / shell，本地输入是否是 200 行以内的小范围结果；超过 200 行是否已改用 Base 云端查询服务查询。 -- 如果使用 `+record-list` / `+record-search`，是否处理了 `has_more`，且投影包含业务 key 和解释字段。 -- 如果涉及关系查询，是否按 `record_id` 或业务 key 精确回查，交付输出是否来自关联表真实字段。 -- 交付输出能追溯到表、字段、筛选条件、排序/聚合条件和连接键。 +### 跨表同类实体与指标 -任一项无法确认时，继续查询或明确说明只能得到局部结论。 +- 多表 users 等重复实体的事实分析，先把各表投影为 `(source_table, source_record_id, entity_id, metric...)` 的 conformed long fact schema，再 `UNION ALL` 并聚合到 entity grain。需要横向比较时，各表先聚合到相同 entity grain 再 JOIN，避免原始事实之间产生 many-to-many fan-out。 +- 没有 Link 时只能使用经过验证的 business key 关联。名称相似匹配属于 entity resolution，不属于普通 JOIN；应作为独立阶段输出匹配依据、置信度和未决项。 diff --git a/skills/lark-base/references/lark-base-data-query-guide.md b/skills/lark-base/references/lark-base-data-query-guide.md index 0ea15ff6fc..c19b0f7d24 100644 --- a/skills/lark-base/references/lark-base-data-query-guide.md +++ b/skills/lark-base/references/lark-base-data-query-guide.md @@ -2,7 +2,7 @@ This guide is the entry point for `+data-query`. Use it for common aggregation fewshots and command selection. For the complete DSL fields, operators, limits, and response details, use [lark-base-data-query.md](lark-base-data-query.md) as the DSL SSOT. -Before using `+data-query`, also follow [lark-base-data-analysis-sop.md](lark-base-data-analysis-sop.md) to confirm that the task really needs aggregation instead of record listing or a temporary view. +For a table-record query or analysis task, first read [lark-base-data-analysis-sop.md](lark-base-data-analysis-sop.md). Read [lark-base-data-analysis-cloud.md](lark-base-data-analysis-cloud.md) and this guide after the SOP enters the Cloud path and selects `+data-query`; read this guide directly when the user explicitly asks about the `+data-query` command or DSL. ## When to use diff --git a/skills/lark-base/references/lark-base-data-query.md b/skills/lark-base/references/lark-base-data-query.md index 2fd3535684..b7d5744bde 100644 --- a/skills/lark-base/references/lark-base-data-query.md +++ b/skills/lark-base/references/lark-base-data-query.md @@ -5,7 +5,7 @@ 本文档是 `+data-query` JSON DSL 的单一事实来源（SSOT），用于说明完整字段、操作符、限制、返回和错误恢复。常用 fewshot 与命令选择先读 [lark-base-data-query-guide.md](lark-base-data-query-guide.md)。 -查询类任务还必须先遵守 [`lark-base-data-analysis-sop.md`](lark-base-data-analysis-sop.md)。`+data-query` 适合让筛选、分组、聚合、排序和 TopN 在 Base 云端查询服务中执行；不要用默认分页的 `+record-list` 或本地 `jq` 替代聚合查询。 +数据表记录查询和分析任务先读 [`lark-base-data-analysis-sop.md`](lark-base-data-analysis-sop.md)；进入 Cloud 路径并选定 `+data-query` 时，读取 [`lark-base-data-analysis-cloud.md`](lark-base-data-analysis-cloud.md) 和本文。用户直接询问 `+data-query` 命令、DSL 或 API 时可直接读取本文。`+data-query` 让筛选、分组、聚合、排序和 TopN 在 Base 云端查询服务中执行。 ## 限制 @@ -432,12 +432,12 @@ CLI 输出标准信封 `{ok, identity, data}`（失败时为 `{ok:false, identit 1. 用 `+data-query` 在 Base 云端查询服务中完成全局筛选、分组、聚合、排序和 TopN，得到业务 key、分组值或候选字段组合。 2. 如果已经拿到候选记录的 `record_id`，用 `+record-get` 读取逐条记录字段。 -3. 如果拿到的是结构化业务 key（例如编号、状态、日期、金额等），用 `+record-list --filter-json` 做精确过滤后读取；不要用 `+record-search` 代替结构化条件。 +3. 如果拿到的是结构化业务 key（例如编号、状态、日期、金额等），用 `+record-list --filter-json` 做精确过滤后读取；`+record-search` 用于文本展示值关键词。 4. 只有候选条件本身是文本展示值关键词时，才使用 `+record-search`，并用 `search_fields` 限定范围、`select_fields` 做投影。 5. 若候选记录包含 link 字段，提取关联 `record_id` 后到关联表用 `+record-get` 批量读取展示字段。 -6. 最终回答业务字段，不要把内部 `record_id` 当作用户可读答案。 +6. 最终回答展示真实业务字段；内部 `record_id` 用于连接或定位。 -不要把 `data-query pagination.limit` 理解为分页扫描；它只限制 Base 云端查询服务返回的聚合结果行数，不支持 offset。需要全量原始记录导出时回到 data analysis SOP 的 `+record-list` 分页规则。 +不要把 `data-query pagination.limit` 理解为分页扫描；它只限制 Base 云端查询服务返回的聚合结果行数，不支持 offset。需要逐条原始记录时按 Cloud SOP 的 `+record-list` / `+record-search` 回查规则处理。 ## 坑点 @@ -450,12 +450,11 @@ CLI 输出标准信封 `{ok, identity, data}`（失败时为 `{ok:false, identit - ⚠️ **数据表标识 `tableId` vs `tableName`**：datasource 中可以用 `tableId`（如 `tblXXX`）或 `tableName`（数据表的用户自定义显示名称），二选一，不要混用 - ⚠️ **`pagination.limit` 最大 5000**：超过会报错，且不支持 offset，只支持 limit - ⚠️ **所有 alias 必须全局唯一**：dimensions 和 measures 之间的 alias 也不能重名 -- ⚠️ **不要用本地分页结果替代 data-query**：凡是全局计数、分组、聚合、排序 TopN，优先让 `+data-query` 在 Base 云端查询服务中执行；默认页 `+record-list` 后本地统计只能得到已读取范围内的结果 ## 参考 - [lark-base](../SKILL.md) — 多维表格全部命令 - [lark-shared](../../lark-shared/SKILL.md) — 认证和全局参数 -- [lark-base-data-analysis-sop.md](lark-base-data-analysis-sop.md) — 查询范围、选路、下推、分页、`+record-list` / `+record-search` 回查和关系查询 SOP +- [lark-base-data-analysis-cloud.md](lark-base-data-analysis-cloud.md) — Cloud 路径的查询范围、下推、分页、`+record-list` / `+record-search` 回查和关系查询 SOP - [lark-base-cell-value.md](lark-base-cell-value.md) — CellValue 格式规范 - [lark-base-field-json.md](lark-base-field-json.md) — 字段类型与 JSON 结构 diff --git a/skills/lark-base/references/lark-base-field-json.md b/skills/lark-base/references/lark-base-field-json.md index 5667ca0bf4..6a1c0fee4e 100644 --- a/skills/lark-base/references/lark-base-field-json.md +++ b/skills/lark-base/references/lark-base-field-json.md @@ -264,7 +264,7 @@ { "type": "datetime", "name": "截止时间", - "default_value": "2026-03-24 10:00:00" + "default_value": "2026-03-24 10:00" } ``` @@ -272,7 +272,7 @@ 默认值 / 约束： - `style.format` 默认 `yyyy/MM/dd` 可用格式：`yyyy/MM/dd`、`yyyy/MM/dd HH:mm`、`yyyy/MM/dd HH:mm Z`、`yyyy-MM-dd`、`yyyy-MM-dd HH:mm`、`yyyy-MM-dd HH:mm Z`、`MM-dd`、`MM/dd/yyyy`、`dd/MM/yyyy` -- `style.format` 只控制前端显示格式；当前可配置格式最多显示到分钟，底层时间值仍可保留秒级精度。 +- `style.format` 只控制 Base 前端展示，不影响 CLI 读取的 CellValue；前端当前最多配置到分钟级展示，底层时间值以毫秒级精度存储。 常用写法： diff --git a/skills/lark-base/references/lark-base-record-upsert.md b/skills/lark-base/references/lark-base-record-upsert.md index 30c28a8caa..024d32e043 100644 --- a/skills/lark-base/references/lark-base-record-upsert.md +++ b/skills/lark-base/references/lark-base-record-upsert.md @@ -13,7 +13,7 @@ lark-cli base +record-upsert --base-token --table-id \ # 更新记录 lark-cli base +record-upsert --base-token --table-id --record-id \ - --json '{"项目名称":"Apollo","状态":"完成","完成时间":"2026-03-24 10:00:00"}' + --json '{"项目名称":"Apollo","状态":"完成","完成时间":"2026-03-24 10:00"}' ``` ## 参数 @@ -42,7 +42,7 @@ lark-cli base +record-upsert --base-token --table-id --r { "项目名称": "Apollo", "状态": "进行中", - "完成时间": "2026-03-24 10:00:00" + "完成时间": "2026-03-24 10:00" } ``` diff --git a/tests/cli_e2e/base/base_record_list_dryrun_test.go b/tests/cli_e2e/base/base_record_list_dryrun_test.go index ba905129b2..c59888c4f1 100644 --- a/tests/cli_e2e/base/base_record_list_dryrun_test.go +++ b/tests/cli_e2e/base/base_record_list_dryrun_test.go @@ -115,6 +115,59 @@ func TestBaseRecordListDryRunTreatsLeadingAtFieldNameLiterally(t *testing.T) { require.Equal(t, "/open-apis/base/v3/bases/app_x/tables/tbl_x/records?field_id=%40Owner&limit=3&offset=0", gjson.Get(result.Stdout, "data.api.0.url").String(), result.Stdout) } +func TestBaseRecordListDryRunInfersNDJSONAndCapsFirstPage(t *testing.T) { + result := runBaseDryRun(t, 0, + "base", "+record-list", + "--base-token", "app_x", + "--table-id", "tbl_x", + "--offset", "50", + "--limit", "2000", + "--output", "exports/records.ndjson", + ) + + out := result.Stdout + require.Equal(t, "/open-apis/base/v3/bases/app_x/tables/tbl_x/records?limit=200&offset=50", gjson.Get(out, "data.api.0.url").String(), out) + require.Equal(t, "ndjson", gjson.Get(out, "data.export_format").String(), out) + require.Equal(t, int64(2000), gjson.Get(out, "data.requested_limit").Int(), out) + require.Equal(t, "exports/records.ndjson", gjson.Get(out, "data.output").String(), out) +} + +func TestBaseRecordSearchDryRunNDJSONCapsFirstPageAndKeepsQuery(t *testing.T) { + result := runBaseDryRun(t, 0, + "base", "+record-search", + "--base-token", "app_x", + "--table-id", "tbl_x", + "--keyword", "Alice", + "--search-field", "Name", + "--offset", "25", + "--limit", "500", + "--output", "search.ndjson", + ) + + out := result.Stdout + require.Equal(t, int64(25), gjson.Get(out, "data.api.0.body.offset").Int(), out) + require.Equal(t, int64(200), gjson.Get(out, "data.api.0.body.limit").Int(), out) + require.Equal(t, "Alice", gjson.Get(out, "data.api.0.body.keyword").String(), out) + require.Equal(t, "ndjson", gjson.Get(out, "data.export_format").String(), out) + require.Equal(t, int64(500), gjson.Get(out, "data.requested_limit").Int(), out) +} + +func TestBaseRecordGetDryRunInfersNDJSON(t *testing.T) { + result := runBaseDryRun(t, 0, + "base", "+record-get", + "--base-token", "app_x", + "--table-id", "tbl_x", + "--record-id", "rec_x", + "--output", "record.ndjson", + ) + + out := result.Stdout + require.Equal(t, "POST", gjson.Get(out, "data.api.0.method").String(), out) + require.Equal(t, "rec_x", gjson.Get(out, "data.api.0.body.record_id_list.0").String(), out) + require.Equal(t, "ndjson", gjson.Get(out, "data.export_format").String(), out) + require.Equal(t, "record.ndjson", gjson.Get(out, "data.output").String(), out) +} + func TestBaseRecordSearchDryRunJSONConflictReportsActualParams(t *testing.T) { result := runBaseDryRun(t, 2, "base", "+record-search", diff --git a/tests/cli_e2e/base/base_skill_contract_test.go b/tests/cli_e2e/base/base_skill_contract_test.go deleted file mode 100644 index f019c33596..0000000000 --- a/tests/cli_e2e/base/base_skill_contract_test.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) 2026 Lark Technologies Pte. Ltd. -// SPDX-License-Identifier: MIT - -package base - -import ( - "path/filepath" - "runtime" - "testing" - - "github.com/larksuite/cli/internal/vfs" - "github.com/stretchr/testify/require" -) - -func TestBaseSkillRoutesFileImportExportToDrive(t *testing.T) { - _, currentFile, _, ok := runtime.Caller(0) - require.True(t, ok) - - skillPath := filepath.Join(filepath.Dir(currentFile), "..", "..", "..", "skills", "lark-base", "SKILL.md") - content, err := vfs.ReadFile(skillPath) - require.NoError(t, err) - - skill := string(content) - require.Contains(t, skill, "文件导入/导出转 lark-drive") - require.Contains(t, skill, "本地文件与 Base 之间的导入/导出转 `lark-drive`") - require.Contains(t, skill, "在线复制走 `+base-copy`") - require.NotContains(t, skill, "--only-schema") - require.NotContains(t, skill, "--output-dir") - require.NotContains(t, skill, "/tmp/") -}