From bc75bfbb7dc1a992181229085a128a7bd46b8e4e Mon Sep 17 00:00:00 2001 From: Abhishek Mishra Date: Mon, 13 Jul 2026 15:54:06 +0530 Subject: [PATCH 1/3] STT fixes: route VAD events by type discriminator, add missing streaming session params, retry-safe transcribe body - Fix WS message dispatch: speech_ended events were delivered to the speech_started handler (presence-based checks could not tell them apart); route on the type field value instead. Verified against the live API. - Add typed streaming connect options from the current Pulse WS API: sentenceTimestamps, redactPii, redactPci, format, punctuate, capitalize, eouTimeoutMs, endpointing, diarize, keywords, itnNormalize, finalizeOnWords, maxWords. additionalProperty(...) entries are now forwarded as query params as an escape hatch for future params. - transcribePulse: use a byte[]-backed request body so the RetryInterceptor can re-send it; the previous one-shot InputStream body sent an empty body on 408/429/5xx retries (sync + async clients). - Guard disconnect() against NPE when connect() was never called. - examples/AzureStyleSttRecognizer.java: recognizing/recognized/canceled event-style adapter for teams migrating from the Azure Speech SDK. --- README.md | 10 + examples/AzureStyleSttRecognizer.java | 126 ++++++ .../resources/waves/AsyncRawWavesClient.java | 4 +- .../java/resources/waves/RawWavesClient.java | 4 +- .../PulseSttStreamingConnectOptions.java | 416 +++++++++++++++++- .../PulseSttStreamingWebSocketClient.java | 59 ++- 6 files changed, 611 insertions(+), 8 deletions(-) create mode 100644 examples/AzureStyleSttRecognizer.java diff --git a/README.md b/README.md index 070d470..adb5089 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,8 @@ stt.connect(PulseSttStreamingConnectOptions.builder() .language("en") .encoding(PulseStreamEncoding.MULAW) // G.711 u-law telephony; LINEAR16 for raw PCM16 .sampleRate(8000) // 8000 for telephony + .eouTimeoutMs(500) // faster end-of-utterance for agent turn-taking + .keywords("Smallest:6,Pulse:6") // boost domain terms (streaming only) .build()).get(); // feed audio as it arrives (e.g. per RTP packet), then finalize @@ -114,8 +116,16 @@ stt.transcribeStreamingPulseSendFinalize( .type(PulseFinalizeSignalMessageType.FINALIZE).build()).get(); ``` +Other session options on the builder: `sentenceTimestamps`, `diarize`, `redactPii`, `redactPci`, +`punctuate`, `capitalize`, `format`, `itnNormalize`, `endpointing`, `finalizeOnWords`, `maxWords`, +`wordTimestamps`, `vadEvents`. New server params not yet typed can be passed with +`.additionalProperty("name", value)` — they are forwarded as query parameters. + A full runnable telephony (8 kHz μ-law) example is in [`examples/StreamingSttTelephony.java`](examples/StreamingSttTelephony.java). +Migrating from Azure Speech SDK? See +[`examples/AzureStyleSttRecognizer.java`](examples/AzureStyleSttRecognizer.java) for a +recognizing/recognized/canceled event-style wrapper. ## Status diff --git a/examples/AzureStyleSttRecognizer.java b/examples/AzureStyleSttRecognizer.java new file mode 100644 index 0000000..2d53434 --- /dev/null +++ b/examples/AzureStyleSttRecognizer.java @@ -0,0 +1,126 @@ +package examples; + +import ai.smallest.SmallestAI; +import ai.smallest.core.DisconnectReason; +import ai.smallest.resources.waves.pulsesttstreaming.types.PulseCloseStreamSignalMessage; +import ai.smallest.resources.waves.pulsesttstreaming.types.PulseCloseStreamSignalMessageType; +import ai.smallest.resources.waves.pulsesttstreaming.types.PulseStreamEncoding; +import ai.smallest.resources.waves.pulsesttstreaming.websocket.PulseSttStreamingConnectOptions; +import ai.smallest.resources.waves.pulsesttstreaming.websocket.PulseSttStreamingWebSocketClient; +import okio.ByteString; + +import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; + +/** + * Drop-in event-style wrapper for teams migrating from the Azure Speech SDK. + * + * Azure's SpeechRecognizer exposes recognizing / recognized / canceled / + * sessionStopped events. This adapter maps the Smallest Pulse streaming STT + * WebSocket onto the same shape so an existing Azure integration can switch + * providers with minimal code movement: + * + * Azure This adapter + * ----- ------------ + * recognizer.recognizing recognizer.recognizing(text -> ...) // interim + * recognizer.recognized recognizer.recognized(text -> ...) // is_final + * recognizer.canceled recognizer.canceled(err -> ...) + * recognizer.sessionStopped recognizer.sessionStopped(r -> ...) + * PushAudioInputStream.write recognizer.writeAudio(bytes) + * startContinuousRecognition recognizer.start() + * stopContinuousRecognition recognizer.stop() + * + * Usage (telephony, G.711 u-law 8 kHz): + * + * var recognizer = new AzureStyleSttRecognizer( + * SmallestAI.builder().build(), + * AzureStyleSttRecognizer.Config.telephony("en")); + * recognizer.recognizing(text -> render(text)); + * recognizer.recognized(text -> handleFinal(text)); + * recognizer.canceled(err -> log(err)); + * recognizer.start().join(); + * // per RTP packet: + * recognizer.writeAudio(packetBytes); + * // on hangup: + * recognizer.stop(); + */ +public final class AzureStyleSttRecognizer implements AutoCloseable { + + public record Config(String language, PulseStreamEncoding encoding, int sampleRate, + Integer eouTimeoutMs, String keywords) { + /** 8 kHz G.711 u-law, tuned for fast agent turn-taking. */ + public static Config telephony(String language) { + return new Config(language, PulseStreamEncoding.MULAW, 8000, 500, null); + } + + /** 16 kHz PCM16, e.g. microphone or decoded audio. */ + public static Config pcm16k(String language) { + return new Config(language, PulseStreamEncoding.LINEAR16, 16000, null, null); + } + } + + private final PulseSttStreamingWebSocketClient ws; + private final Config config; + + private volatile Consumer recognizing = t -> {}; + private volatile Consumer recognized = t -> {}; + private volatile Consumer canceled = e -> {}; + private volatile Consumer sessionStopped = r -> {}; + + public AzureStyleSttRecognizer(SmallestAI client, Config config) { + this.config = config; + this.ws = client.waves().pulseSttStreaming().pulseSttStreamingWebSocket(); + + ws.receiveTranscribeStreamingPulse(m -> { + String text = m.getTranscript().orElse(""); + if (m.getIsFinal().orElse(false)) { + if (!text.isBlank()) recognized.accept(text); + } else { + recognizing.accept(text); + } + }); + ws.onError(e -> canceled.accept(e)); + ws.onDisconnected(r -> sessionStopped.accept(r)); + } + + public void recognizing(Consumer handler) { this.recognizing = handler; } + + public void recognized(Consumer handler) { this.recognized = handler; } + + public void canceled(Consumer handler) { this.canceled = handler; } + + public void sessionStopped(Consumer handler) { this.sessionStopped = handler; } + + /** Equivalent of startContinuousRecognitionAsync(). */ + public CompletableFuture start() { + var opts = PulseSttStreamingConnectOptions.builder() + .language(config.language()) + .encoding(config.encoding()) + .sampleRate(config.sampleRate()); + if (config.eouTimeoutMs() != null) opts.eouTimeoutMs(config.eouTimeoutMs()); + if (config.keywords() != null) opts.keywords(config.keywords()); + return ws.connect(opts.build()); + } + + /** Equivalent of PushAudioInputStream.write(). Push audio as it arrives. */ + public void writeAudio(byte[] audio) { + ws.transcribeStreamingPulseSendAudio(ByteString.of(audio)); + } + + /** + * Equivalent of stopContinuousRecognitionAsync(): asks the server to flush and + * deliver the last transcript (is_last=true), then closes the socket. + */ + public void stop() throws Exception { + ws.transcribeStreamingPulseSendCloseStream(PulseCloseStreamSignalMessage.builder() + .type(PulseCloseStreamSignalMessageType.CLOSE_STREAM).build()).get(); + // Give the server a moment to deliver the final transcript before closing. + Thread.sleep(1000); + ws.close(); + } + + @Override + public void close() throws Exception { + stop(); + } +} diff --git a/src/main/java/resources/waves/AsyncRawWavesClient.java b/src/main/java/resources/waves/AsyncRawWavesClient.java index 61681bd..172273c 100644 --- a/src/main/java/resources/waves/AsyncRawWavesClient.java +++ b/src/main/java/resources/waves/AsyncRawWavesClient.java @@ -2514,7 +2514,9 @@ public CompletableFuture> t httpUrl.addQueryParameter(_key, _value); } ); } - RequestBody body = new InputStreamRequestBody(MediaType.parse("application/octet-stream"), new ByteArrayInputStream(request.getBody())); + // byte[]-backed body so the RetryInterceptor can re-send it; a one-shot + // InputStream body would be empty on the retry attempt + RequestBody body = RequestBody.create(request.getBody(), MediaType.parse("application/octet-stream")); Request.Builder _requestBuilder = new Request.Builder() .url(httpUrl.build()) .method("POST", body) diff --git a/src/main/java/resources/waves/RawWavesClient.java b/src/main/java/resources/waves/RawWavesClient.java index 396c2ed..a287240 100644 --- a/src/main/java/resources/waves/RawWavesClient.java +++ b/src/main/java/resources/waves/RawWavesClient.java @@ -2204,7 +2204,9 @@ public SmallestAIHttpResponse transcribePulse( httpUrl.addQueryParameter(_key, _value); } ); } - RequestBody body = new InputStreamRequestBody(MediaType.parse("application/octet-stream"), new ByteArrayInputStream(request.getBody())); + // byte[]-backed body so the RetryInterceptor can re-send it; a one-shot + // InputStream body would be empty on the retry attempt + RequestBody body = RequestBody.create(request.getBody(), MediaType.parse("application/octet-stream")); Request.Builder _requestBuilder = new Request.Builder() .url(httpUrl.build()) .method("POST", body) diff --git a/src/main/java/resources/waves/pulsesttstreaming/websocket/PulseSttStreamingConnectOptions.java b/src/main/java/resources/waves/pulsesttstreaming/websocket/PulseSttStreamingConnectOptions.java index 1aa92c0..7921301 100644 --- a/src/main/java/resources/waves/pulsesttstreaming/websocket/PulseSttStreamingConnectOptions.java +++ b/src/main/java/resources/waves/pulsesttstreaming/websocket/PulseSttStreamingConnectOptions.java @@ -16,6 +16,7 @@ import com.fasterxml.jackson.annotation.JsonSetter; import com.fasterxml.jackson.annotation.Nulls; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import java.lang.Boolean; import java.lang.Integer; import java.lang.Object; import java.lang.String; @@ -39,17 +40,62 @@ public final class PulseSttStreamingConnectOptions { private final Optional vadEvents; + private final Optional sentenceTimestamps; + + private final Optional redactPii; + + private final Optional redactPci; + + private final Optional format; + + private final Optional punctuate; + + private final Optional capitalize; + + private final Optional eouTimeoutMs; + + private final Optional endpointing; + + private final Optional diarize; + + private final Optional keywords; + + private final Optional itnNormalize; + + private final Optional finalizeOnWords; + + private final Optional maxWords; + private final Map additionalProperties; private PulseSttStreamingConnectOptions(Optional language, Optional encoding, Optional sampleRate, Optional wordTimestamps, Optional vadEvents, + Optional sentenceTimestamps, Optional redactPii, + Optional redactPci, Optional format, Optional punctuate, + Optional capitalize, Optional eouTimeoutMs, Optional endpointing, + Optional diarize, + Optional keywords, Optional itnNormalize, + Optional finalizeOnWords, Optional maxWords, Map additionalProperties) { this.language = language; this.encoding = encoding; this.sampleRate = sampleRate; this.wordTimestamps = wordTimestamps; this.vadEvents = vadEvents; + this.sentenceTimestamps = sentenceTimestamps; + this.redactPii = redactPii; + this.redactPci = redactPci; + this.format = format; + this.punctuate = punctuate; + this.capitalize = capitalize; + this.eouTimeoutMs = eouTimeoutMs; + this.endpointing = endpointing; + this.diarize = diarize; + this.keywords = keywords; + this.itnNormalize = itnNormalize; + this.finalizeOnWords = finalizeOnWords; + this.maxWords = maxWords; this.additionalProperties = additionalProperties; } @@ -93,6 +139,110 @@ public Optional getVadEvents() { return vadEvents; } + /** + * @return Include sentence-level timestamps (utterances) in transcription. + */ + @JsonProperty("sentence_timestamps") + public Optional getSentenceTimestamps() { + return sentenceTimestamps; + } + + /** + * @return Redact personally identifiable information (name, address, etc.) in finalized transcripts. + */ + @JsonProperty("redact_pii") + public Optional getRedactPii() { + return redactPii; + } + + /** + * @return Redact payment card information (card number, CVV, etc.) in finalized transcripts. + */ + @JsonProperty("redact_pci") + public Optional getRedactPci() { + return redactPci; + } + + /** + * @return Master formatting switch. When false, forces punctuate=false, capitalize=false and disables ITN. + */ + @JsonProperty("format") + public Optional getFormat() { + return format; + } + + /** + * @return When false, strips end-of-sentence punctuation from finalized transcripts. + */ + @JsonProperty("punctuate") + public Optional getPunctuate() { + return punctuate; + } + + /** + * @return When false, lowercases the entire transcript output. + */ + @JsonProperty("capitalize") + public Optional getCapitalize() { + return capitalize; + } + + /** + * @return Milliseconds of trailing silence before the transcript is flushed as final (default 800, range 100-10000). Lower for fast voice-agent turn-taking. + */ + @JsonProperty("eou_timeout_ms") + public Optional getEouTimeoutMs() { + return eouTimeoutMs; + } + + /** + * @return Finalize transcripts on trailing-silence detection (default true). eou_timeout_ms acts as the ceiling when enabled. + */ + @JsonProperty("endpointing") + public Optional getEndpointing() { + return endpointing; + } + + /** + * @return Enable speaker diarization to identify different speakers in the audio. + */ + @JsonProperty("diarize") + public Optional getDiarize() { + return diarize; + } + + /** + * @return Keyword boosting for this session (WebSocket only). Comma-separated WORD:INTENSIFIER entries, e.g. NVIDIA:6,Jensen. Intensifier defaults to 1.0; recommended around 3-6. + */ + @JsonProperty("keywords") + public Optional getKeywords() { + return keywords; + } + + /** + * @return Enable Inverse Text Normalization (numbers, dates, currencies to written form) in finalized transcripts. + */ + @JsonProperty("itn_normalize") + public Optional getItnNormalize() { + return itnNormalize; + } + + /** + * @return When false, disables automatic word-count-based finalization; control finalization via the finalize message. + */ + @JsonProperty("finalize_on_words") + public Optional getFinalizeOnWords() { + return finalizeOnWords; + } + + /** + * @return Maximum number of words before forced finalization. + */ + @JsonProperty("max_words") + public Optional getMaxWords() { + return maxWords; + } + @java.lang.Override public boolean equals(Object other) { if (this == other) return true; @@ -105,12 +255,12 @@ public Map getAdditionalProperties() { } private boolean equalTo(PulseSttStreamingConnectOptions other) { - return language.equals(other.language) && encoding.equals(other.encoding) && sampleRate.equals(other.sampleRate) && wordTimestamps.equals(other.wordTimestamps) && vadEvents.equals(other.vadEvents); + return language.equals(other.language) && encoding.equals(other.encoding) && sampleRate.equals(other.sampleRate) && wordTimestamps.equals(other.wordTimestamps) && vadEvents.equals(other.vadEvents) && sentenceTimestamps.equals(other.sentenceTimestamps) && redactPii.equals(other.redactPii) && redactPci.equals(other.redactPci) && format.equals(other.format) && punctuate.equals(other.punctuate) && capitalize.equals(other.capitalize) && eouTimeoutMs.equals(other.eouTimeoutMs) && endpointing.equals(other.endpointing) && diarize.equals(other.diarize) && keywords.equals(other.keywords) && itnNormalize.equals(other.itnNormalize) && finalizeOnWords.equals(other.finalizeOnWords) && maxWords.equals(other.maxWords); } @java.lang.Override public int hashCode() { - return Objects.hash(this.language, this.encoding, this.sampleRate, this.wordTimestamps, this.vadEvents); + return Objects.hash(this.language, this.encoding, this.sampleRate, this.wordTimestamps, this.vadEvents, this.sentenceTimestamps, this.redactPii, this.redactPci, this.format, this.punctuate, this.capitalize, this.eouTimeoutMs, this.endpointing, this.diarize, this.keywords, this.itnNormalize, this.finalizeOnWords, this.maxWords); } @java.lang.Override @@ -136,6 +286,32 @@ public static final class Builder { private Optional vadEvents = Optional.empty(); + private Optional sentenceTimestamps = Optional.empty(); + + private Optional redactPii = Optional.empty(); + + private Optional redactPci = Optional.empty(); + + private Optional format = Optional.empty(); + + private Optional punctuate = Optional.empty(); + + private Optional capitalize = Optional.empty(); + + private Optional eouTimeoutMs = Optional.empty(); + + private Optional endpointing = Optional.empty(); + + private Optional diarize = Optional.empty(); + + private Optional keywords = Optional.empty(); + + private Optional itnNormalize = Optional.empty(); + + private Optional finalizeOnWords = Optional.empty(); + + private Optional maxWords = Optional.empty(); + @JsonAnySetter private Map additionalProperties = new HashMap<>(); @@ -148,6 +324,19 @@ public Builder from(PulseSttStreamingConnectOptions other) { sampleRate(other.getSampleRate()); wordTimestamps(other.getWordTimestamps()); vadEvents(other.getVadEvents()); + sentenceTimestamps(other.getSentenceTimestamps()); + redactPii(other.getRedactPii()); + redactPci(other.getRedactPci()); + format(other.getFormat()); + punctuate(other.getPunctuate()); + capitalize(other.getCapitalize()); + eouTimeoutMs(other.getEouTimeoutMs()); + endpointing(other.getEndpointing()); + diarize(other.getDiarize()); + keywords(other.getKeywords()); + itnNormalize(other.getItnNormalize()); + finalizeOnWords(other.getFinalizeOnWords()); + maxWords(other.getMaxWords()); return this; } @@ -236,8 +425,229 @@ public Builder vadEvents(PulseStreamVadEvents vadEvents) { return this; } + /** + *

Include sentence-level timestamps (utterances) in transcription.

+ */ + @JsonSetter( + value = "sentence_timestamps", + nulls = Nulls.SKIP + ) + public Builder sentenceTimestamps(Optional sentenceTimestamps) { + this.sentenceTimestamps = sentenceTimestamps; + return this; + } + + public Builder sentenceTimestamps(Boolean sentenceTimestamps) { + this.sentenceTimestamps = Optional.ofNullable(sentenceTimestamps); + return this; + } + + /** + *

Redact personally identifiable information in finalized transcripts.

+ */ + @JsonSetter( + value = "redact_pii", + nulls = Nulls.SKIP + ) + public Builder redactPii(Optional redactPii) { + this.redactPii = redactPii; + return this; + } + + public Builder redactPii(Boolean redactPii) { + this.redactPii = Optional.ofNullable(redactPii); + return this; + } + + /** + *

Redact payment card information in finalized transcripts.

+ */ + @JsonSetter( + value = "redact_pci", + nulls = Nulls.SKIP + ) + public Builder redactPci(Optional redactPci) { + this.redactPci = redactPci; + return this; + } + + public Builder redactPci(Boolean redactPci) { + this.redactPci = Optional.ofNullable(redactPci); + return this; + } + + /** + *

Master formatting switch. When false, forces punctuate=false, capitalize=false and disables ITN.

+ */ + @JsonSetter( + value = "format", + nulls = Nulls.SKIP + ) + public Builder format(Optional format) { + this.format = format; + return this; + } + + public Builder format(Boolean format) { + this.format = Optional.ofNullable(format); + return this; + } + + /** + *

When false, strips end-of-sentence punctuation from finalized transcripts.

+ */ + @JsonSetter( + value = "punctuate", + nulls = Nulls.SKIP + ) + public Builder punctuate(Optional punctuate) { + this.punctuate = punctuate; + return this; + } + + public Builder punctuate(Boolean punctuate) { + this.punctuate = Optional.ofNullable(punctuate); + return this; + } + + /** + *

When false, lowercases the entire transcript output.

+ */ + @JsonSetter( + value = "capitalize", + nulls = Nulls.SKIP + ) + public Builder capitalize(Optional capitalize) { + this.capitalize = capitalize; + return this; + } + + public Builder capitalize(Boolean capitalize) { + this.capitalize = Optional.ofNullable(capitalize); + return this; + } + + /** + *

Milliseconds of trailing silence before the transcript is flushed as final (default 800, range 100-10000).

+ */ + @JsonSetter( + value = "eou_timeout_ms", + nulls = Nulls.SKIP + ) + public Builder eouTimeoutMs(Optional eouTimeoutMs) { + this.eouTimeoutMs = eouTimeoutMs; + return this; + } + + public Builder eouTimeoutMs(Integer eouTimeoutMs) { + this.eouTimeoutMs = Optional.ofNullable(eouTimeoutMs); + return this; + } + + /** + *

Finalize transcripts on trailing-silence detection (default true).

+ */ + @JsonSetter( + value = "endpointing", + nulls = Nulls.SKIP + ) + public Builder endpointing(Optional endpointing) { + this.endpointing = endpointing; + return this; + } + + public Builder endpointing(Boolean endpointing) { + this.endpointing = Optional.ofNullable(endpointing); + return this; + } + + /** + *

Enable speaker diarization to identify different speakers in the audio.

+ */ + @JsonSetter( + value = "diarize", + nulls = Nulls.SKIP + ) + public Builder diarize(Optional diarize) { + this.diarize = diarize; + return this; + } + + public Builder diarize(Boolean diarize) { + this.diarize = Optional.ofNullable(diarize); + return this; + } + + /** + *

Keyword boosting (WebSocket only). Comma-separated WORD:INTENSIFIER entries, e.g. NVIDIA:6,Jensen.

+ */ + @JsonSetter( + value = "keywords", + nulls = Nulls.SKIP + ) + public Builder keywords(Optional keywords) { + this.keywords = keywords; + return this; + } + + public Builder keywords(String keywords) { + this.keywords = Optional.ofNullable(keywords); + return this; + } + + /** + *

Enable Inverse Text Normalization in finalized transcripts.

+ */ + @JsonSetter( + value = "itn_normalize", + nulls = Nulls.SKIP + ) + public Builder itnNormalize(Optional itnNormalize) { + this.itnNormalize = itnNormalize; + return this; + } + + public Builder itnNormalize(Boolean itnNormalize) { + this.itnNormalize = Optional.ofNullable(itnNormalize); + return this; + } + + /** + *

When false, disables automatic word-count-based finalization.

+ */ + @JsonSetter( + value = "finalize_on_words", + nulls = Nulls.SKIP + ) + public Builder finalizeOnWords(Optional finalizeOnWords) { + this.finalizeOnWords = finalizeOnWords; + return this; + } + + public Builder finalizeOnWords(Boolean finalizeOnWords) { + this.finalizeOnWords = Optional.ofNullable(finalizeOnWords); + return this; + } + + /** + *

Maximum number of words before forced finalization.

+ */ + @JsonSetter( + value = "max_words", + nulls = Nulls.SKIP + ) + public Builder maxWords(Optional maxWords) { + this.maxWords = maxWords; + return this; + } + + public Builder maxWords(Integer maxWords) { + this.maxWords = Optional.ofNullable(maxWords); + return this; + } + public PulseSttStreamingConnectOptions build() { - return new PulseSttStreamingConnectOptions(language, encoding, sampleRate, wordTimestamps, vadEvents, additionalProperties); + return new PulseSttStreamingConnectOptions(language, encoding, sampleRate, wordTimestamps, vadEvents, sentenceTimestamps, redactPii, redactPci, format, punctuate, capitalize, eouTimeoutMs, endpointing, diarize, keywords, itnNormalize, finalizeOnWords, maxWords, additionalProperties); } public Builder additionalProperty(String key, Object value) { diff --git a/src/main/java/resources/waves/pulsesttstreaming/websocket/PulseSttStreamingWebSocketClient.java b/src/main/java/resources/waves/pulsesttstreaming/websocket/PulseSttStreamingWebSocketClient.java index 4e1a9c0..d8faf89 100644 --- a/src/main/java/resources/waves/pulsesttstreaming/websocket/PulseSttStreamingWebSocketClient.java +++ b/src/main/java/resources/waves/pulsesttstreaming/websocket/PulseSttStreamingWebSocketClient.java @@ -124,6 +124,54 @@ else if (baseUrl.startsWith("ws://")) { if (options.getVadEvents() != null && options.getVadEvents().isPresent()) { urlBuilder.addQueryParameter("vad_events", String.valueOf(options.getVadEvents().get())); } + if (options.getSentenceTimestamps() != null && options.getSentenceTimestamps().isPresent()) { + urlBuilder.addQueryParameter("sentence_timestamps", String.valueOf(options.getSentenceTimestamps().get())); + } + if (options.getRedactPii() != null && options.getRedactPii().isPresent()) { + urlBuilder.addQueryParameter("redact_pii", String.valueOf(options.getRedactPii().get())); + } + if (options.getRedactPci() != null && options.getRedactPci().isPresent()) { + urlBuilder.addQueryParameter("redact_pci", String.valueOf(options.getRedactPci().get())); + } + if (options.getFormat() != null && options.getFormat().isPresent()) { + urlBuilder.addQueryParameter("format", String.valueOf(options.getFormat().get())); + } + if (options.getPunctuate() != null && options.getPunctuate().isPresent()) { + urlBuilder.addQueryParameter("punctuate", String.valueOf(options.getPunctuate().get())); + } + if (options.getCapitalize() != null && options.getCapitalize().isPresent()) { + urlBuilder.addQueryParameter("capitalize", String.valueOf(options.getCapitalize().get())); + } + if (options.getEouTimeoutMs() != null && options.getEouTimeoutMs().isPresent()) { + urlBuilder.addQueryParameter("eou_timeout_ms", String.valueOf(options.getEouTimeoutMs().get())); + } + if (options.getEndpointing() != null && options.getEndpointing().isPresent()) { + urlBuilder.addQueryParameter("endpointing", String.valueOf(options.getEndpointing().get())); + } + if (options.getDiarize() != null && options.getDiarize().isPresent()) { + urlBuilder.addQueryParameter("diarize", String.valueOf(options.getDiarize().get())); + } + if (options.getKeywords() != null && options.getKeywords().isPresent()) { + urlBuilder.addQueryParameter("keywords", options.getKeywords().get()); + } + if (options.getItnNormalize() != null && options.getItnNormalize().isPresent()) { + urlBuilder.addQueryParameter("itn_normalize", String.valueOf(options.getItnNormalize().get())); + } + if (options.getFinalizeOnWords() != null && options.getFinalizeOnWords().isPresent()) { + urlBuilder.addQueryParameter("finalize_on_words", String.valueOf(options.getFinalizeOnWords().get())); + } + if (options.getMaxWords() != null && options.getMaxWords().isPresent()) { + urlBuilder.addQueryParameter("max_words", String.valueOf(options.getMaxWords().get())); + } + // Escape hatch: forward any additionalProperty(...) entries as query parameters so + // new server-side session params are usable before they get typed options. + if (options.getAdditionalProperties() != null) { + options.getAdditionalProperties().forEach((key, value) -> { + if (value != null) { + urlBuilder.addQueryParameter(key, String.valueOf(value)); + } + }); + } Request.Builder requestBuilder = new Request.Builder().url(urlBuilder.build()); clientOptions.headers((RequestOptions) null).forEach(requestBuilder::addHeader); final Request request = requestBuilder.build(); @@ -188,7 +236,9 @@ public CompletableFuture connect() { * Disconnects the WebSocket connection and releases resources. */ public void disconnect() { - reconnectingListener.disconnect(); + if (reconnectingListener != null) { + reconnectingListener.disconnect(); + } if (timeoutExecutor != null) { timeoutExecutor.shutdownNow(); timeoutExecutor = null; @@ -360,7 +410,10 @@ private void handleIncomingMessage(String json) { if (node == null || node.isNull()) { throw new IllegalArgumentException("Received null or invalid JSON message"); } - if (node.has("type") + // Dispatch VAD events on the `type` discriminator value; presence checks alone + // cannot distinguish speech_started from speech_ended (identical field sets). + String messageType = node.has("type") ? node.get("type").asText() : null; + if ("speech_started".equals(messageType) && node.has("session_id") && node.has("timestamp")) { PulseSpeechStartedEventMessage receiveSpeechStartedStreamingPulseHandlerEvent = null; @@ -375,7 +428,7 @@ private void handleIncomingMessage(String json) { return; } } - if (node.has("type") + if ("speech_ended".equals(messageType) && node.has("session_id") && node.has("timestamp")) { PulseSpeechEndedEventMessage receiveSpeechEndedStreamingPulseHandlerEvent = null; From e77250962ff51c330fc5119c4cbd29f2c74ccb1f Mon Sep 17 00:00:00 2001 From: Abhishek Mishra Date: Mon, 13 Jul 2026 16:01:15 +0530 Subject: [PATCH 2/3] Document generation/maintenance model: .fernignore for hand-patched files, README section - .fernignore (Python SDK convention) lists hand-written files and the four generated files carrying manual STT fixes, each with its upstream path (spec change in smallest-ai-documentation vs generator bug report to Fern). - README explains the SDK is manually generated (see .fern/metadata.json), not yet in the Fern CI pipeline, and the rules for regenerating safely. - gitignore temp/ (scratch clones used for coverage audits). --- .fernignore | 45 +++++++++++++++++++++++++++++++++++++++++++++ .gitignore | 1 + README.md | 29 ++++++++++++++++++++++++++++- 3 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 .fernignore diff --git a/.fernignore b/.fernignore new file mode 100644 index 0000000..707f19a --- /dev/null +++ b/.fernignore @@ -0,0 +1,45 @@ +# Files Fern must not overwrite on regeneration. +# Mirrors the convention used in smallest-python-sdk/.fernignore. + +# Hand-written repo scaffolding (generator never produces these) +README.md +CONTRIBUTING.md +build.gradle +settings.gradle +jitpack.yml +gradle/** +gradlew +gradlew.bat +.gitignore +.fernignore + +# Hand-written examples +examples/** + +# --- Hand-patched GENERATED files (temporary; upstream before removing) --- +# These four files carry manual fixes that are NOT yet in the Fern spec or +# generator. Regenerating without upstreaming first will silently reintroduce +# the bugs. Tracking: +# +# 1. PulseSttStreamingWebSocketClient.java +# - VAD event routing by `type` discriminator (generator emitted +# presence-based checks that sent speech_ended to the speech_started +# handler). Needs a fern-java-sdk generator fix; report to Fern. +# - Streaming session query params + additionalProperties passthrough in +# connect(). Upstream: expose the full ws query binding from +# fern/apis/waves/asyncapi/pulse-stt-ws(-overrides).yml in +# smallest-ai-documentation so the generator emits these. +# - Null guard in disconnect() when connect() was never called. +# 2. PulseSttStreamingConnectOptions.java +# - Typed options for keywords, eou_timeout_ms, endpointing, +# sentence_timestamps, redact_pii, redact_pci, diarize, format, +# punctuate, capitalize, itn_normalize, finalize_on_words, max_words. +# Same upstream path as above. +# 3. RawWavesClient.java / 4. AsyncRawWavesClient.java +# - transcribePulse uses a byte[]-backed RequestBody so the +# RetryInterceptor can re-send it (one-shot InputStream body sent an +# empty body on 408/429/5xx retries). Generator fix; report to Fern. +src/main/java/resources/waves/pulsesttstreaming/websocket/PulseSttStreamingWebSocketClient.java +src/main/java/resources/waves/pulsesttstreaming/websocket/PulseSttStreamingConnectOptions.java +src/main/java/resources/waves/RawWavesClient.java +src/main/java/resources/waves/AsyncRawWavesClient.java diff --git a/.gitignore b/.gitignore index 8b9bbb4..1267b52 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .gradle/ build/ +temp/ *.class .DS_Store .idea/ diff --git a/README.md b/README.md index adb5089..cf883ce 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,33 @@ Migrating from Azure Speech SDK? See [`examples/AzureStyleSttRecognizer.java`](examples/AzureStyleSttRecognizer.java) for a recognizing/recognized/canceled event-style wrapper. +## Generation and maintenance + +This SDK is generated with `fernapi/fern-java-sdk` from the Fern API definition in +[smallest-ai-documentation](https://github.com/smallest-inc/smallest-ai-documentation) +(`fern/apis/unified`). Generation metadata (generator version, origin spec commit) is +recorded in `.fern/metadata.json`. + +**Current state: generated manually, with hand patches.** The Java SDK is not yet wired +into the Fern CI pipeline (`generators.yml` has no `java-sdk` group), and a small set of +generated files carry manual fixes that are not yet in the spec or generator. The full +list, with the reason for each patch and where it must be upstreamed, lives in +[`.fernignore`](.fernignore) — the same convention the +[Python SDK](https://github.com/smallest-inc/smallest-python-sdk) uses to protect +customer-owned files from regeneration. + +Rules for maintainers: + +- Do not regenerate over the files listed in `.fernignore` until their patches are + upstreamed (spec changes go to `smallest-ai-documentation`; generator bugs go to Fern + support). Regenerating first silently reintroduces known bugs (see `.fernignore`). +- Hand-written files (`examples/`, `build.gradle`, this README, etc.) are always + customer-owned and listed in `.fernignore`. +- End state: add a `java-sdk` group to `fern/apis/unified/generators.yml` with + `github: mode: pull-request` (like the Python SDK), upstream the streaming session + params into the AsyncAPI ws bindings, and shrink `.fernignore` down to the + hand-written files only. + ## Status -Initial `0.1.3` snapshot. Real-time streaming STT/TTS WebSocket clients are included and evolving. +Initial `0.1.x` snapshot. Real-time streaming STT/TTS WebSocket clients are included and evolving. From 75f46883f5a8e62dcb9e046b3699de2dc53b7346 Mon Sep 17 00:00:00 2001 From: Abhishek Mishra Date: Mon, 13 Jul 2026 16:12:07 +0530 Subject: [PATCH 3/3] README: style cleanup in new sections --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cf883ce..877ac86 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ stt.transcribeStreamingPulseSendFinalize( Other session options on the builder: `sentenceTimestamps`, `diarize`, `redactPii`, `redactPci`, `punctuate`, `capitalize`, `format`, `itnNormalize`, `endpointing`, `finalizeOnWords`, `maxWords`, `wordTimestamps`, `vadEvents`. New server params not yet typed can be passed with -`.additionalProperty("name", value)` — they are forwarded as query parameters. +`.additionalProperty("name", value)`: they are forwarded as query parameters. A full runnable telephony (8 kHz μ-law) example is in [`examples/StreamingSttTelephony.java`](examples/StreamingSttTelephony.java). @@ -138,7 +138,7 @@ recorded in `.fern/metadata.json`. into the Fern CI pipeline (`generators.yml` has no `java-sdk` group), and a small set of generated files carry manual fixes that are not yet in the spec or generator. The full list, with the reason for each patch and where it must be upstreamed, lives in -[`.fernignore`](.fernignore) — the same convention the +[`.fernignore`](.fernignore), the same convention the [Python SDK](https://github.com/smallest-inc/smallest-python-sdk) uses to protect customer-owned files from regeneration.