NextToken in response is invalid and not similar to CLI equivalent

[tinkyholloway]

Using the cli:

$ aws stepfunctions list-state-machines --max-items 1 | jq .
{
  "stateMachines": [
    {
      "stateMachineArn": "arn:aws:states:us-east-1:XXXXXXXXXXXX:stateMachine:HelloWorld",
      "name": "HelloWorld",
      "type": "STANDARD",
      "creationDate": "2022-06-10T21:33:12.248000+12:00"
    }
  ],
  "NextToken": "eyJuZXh0VG9rZW4iOiBudWxsLCAiYm90b190cnVuY2F0ZV9hbW91bnQiOiAxfQ=="
}

My attempt with the SDK:

    let r = client
        .list_state_machines()
        .set_max_results(Some(1))
        .set_next_token(None)
        .send()
        .await?;
    println!("{:?}", r);

gives

ListStateMachinesOutput { 
    state_machines: Some(<snip>),
    next_token: Some("AAAAKgAAAAIAAAAAAAAAAXjMVkCgxyAlq9I9C4kWH/PVZN2xGUVSKleVGzock3k643DfUGoYMVa0yLSWW2J1rNkdAIxXsBniGNsGxS+7yQBCzirKJGcsZr3ziUoaTo/HWFz8SX82XRp7eEgbkhPMxFB3WTMXgxu3sxHk835oiWwkYjmZtLSFz22icv/GPrbdUNC4K0=") 
}

The returned next_token here looks quite different from that returned by the cli and and when I use it in a subsequent call to get the next page of data, I get an error back that says the token is invalid.

Questions:

1. can you see what I am doing wrong in my use the the SDK (version 0.13.0)? Do I need to transform the returned next_token before I use it?
2. is there a way to turn on logging in the SDK so that I can see the requests and responses?

[rcoh]

interesting—I don't think you need to transform the next token. Did you try using the paginator? https://docs.rs/aws-sdk-sfn/latest/aws_sdk_sfn/client/struct.Client.html#method.list_state_machines

use tokio_stream::StreamExt;
let items = client.list_step_functions().max_results(1).into_paginator().items().send().await;
let state_machines = items.collect::<Result<Vec<_>, _>>().await?

That said, I would be a little surprised if that worked? I don't think there is any transformation that occurs.

You can enable logging with the instructions here: https://docs.aws.amazon.com/sdk-for-rust/latest/dg/logging.html

[tinkyholloway]

    let r = client
        .list_state_machines()
        .max_results(1)
        .into_paginator()
        .items()
        .send();
    let state_machines = r.collect::<Result<Vec<_>, _>>().await;
    println!("{:?}", state_machines);

gives

2022-06-17T06:49:57.874846Z TRACE send_operation{operation="ListStateMachines" service="sfn"}: aws_smithy_http_tower::dispatch: request=Request { method: POST, uri: https://states.us-east-1.amazonaws.com/, version: HTTP/1.1, headers: {"content-type": "application/x-amz-json-1.0", "x-amz-target": "AWSStepFunctions.ListStateMachines", "content-length": "16", "user-agent": "aws-sdk-rust/0.13.0 os/macos lang/rust/1.63.0-nightly", "x-amz-user-agent": "aws-sdk-rust/0.13.0 api/sfn/0.13.0 os/macos lang/rust/1.63.0-nightly", "x-amz-date": "20220617T064957Z", "authorization": Sensitive}, body: SdkBody { inner: Once(Some(b"{\"maxResults\":1}")), retryable: true } }
2022-06-17T06:49:58.876358Z TRACE send_operation{operation="ListStateMachines" service="sfn"}:load_response: aws_smithy_http::middleware: http_response=Response { status: 200, version: HTTP/1.1, headers: {"x-amzn-requestid": "f314249d-abf5-4be1-91ae-b2ae3cfadf8b", "date": "Fri, 17 Jun 2022 06:49:58 GMT", "content-type": "application/x-amz-json-1.0", "content-length": "424"}, body: b"{\"nextToken\":\"AAAAKgAAAAIAAAAAAAAAAfD6nnzK3s/NcMDbc3Cw7LFKwYWUawhWIUS896FQO/8BGpjIasscAmau68entloBo7pXaqMJq9yRJkhkS3ajtYG7JE+p5UE3CC4NZF4Y/zNEKm2Rs2IXgX3hzBSlo0GT1Qcnd4sfgkx0q3OFfjX2t1lfCfyM3HShWDff9gB/195zLnCZdQHvGlAVoTEDfib7uOJohqWry9qC0DzHoQIULyQ=\",\"stateMachines\":[{\"creationDate\":1.654853592248E9,\"name\":\"HelloWorld\",\"stateMachineArn\":\"arn:aws:states:us-east-1:XXXXXXXXXXXX:stateMachine:HelloWorld\",\"type\":\"STANDARD\"}]}" }
2022-06-17T06:49:58.879443Z TRACE send_operation{operation="ListStateMachines" service="sfn"}: aws_smithy_http_tower::dispatch: request=Request { method: POST, uri: https://states.us-east-1.amazonaws.com/, version: HTTP/1.1, headers: {"content-type": "application/x-amz-json-1.0", "x-amz-target": "AWSStepFunctions.ListStateMachines", "content-length": "267", "user-agent": "aws-sdk-rust/0.13.0 os/macos lang/rust/1.63.0-nightly", "x-amz-user-agent": "aws-sdk-rust/0.13.0 api/sfn/0.13.0 os/macos lang/rust/1.63.0-nightly", "x-amz-date": "20220617T064958Z", "authorization": Sensitive}, body: SdkBody { inner: Once(Some(b"{\"maxResults\":1,\"nextToken\":\"AAAAKgAAAAIAAAAAAAAAAfD6nnzK3s/NcMDbc3Cw7LFKwYWUawhWIUS896FQO/8BGpjIasscAmau68entloBo7pXaqMJq9yRJkhkS3ajtYG7JE+p5UE3CC4NZF4Y/zNEKm2Rs2IXgX3hzBSlo0GT1Qcnd4sfgkx0q3OFfjX2t1lfCfyM3HShWDff9gB/195zLnCZdQHvGlAVoTEDfib7uOJohqWry9qC0DzHoQIULyQ=\"}")), retryable: true } }
2022-06-17T06:49:59.124697Z TRACE send_operation{operation="ListStateMachines" service="sfn"}:load_response: aws_smithy_http::middleware: http_response=Response { status: 200, version: HTTP/1.1, headers: {"x-amzn-requestid": "82d90014-aaf4-494d-84ff-6892308598fb", "date": "Fri, 17 Jun 2022 06:49:59 GMT", "content-type": "application/x-amz-json-1.0", "content-length": "180"}, body: b"{\"stateMachines\":[{\"creationDate\":1.65537237346E9,\"name\":\"MyStateMachine\",\"stateMachineArn\":\"arn:aws:states:us-east-1:XXXXXXXXXXXX:stateMachine:MyStateMachine\",\"type\":\"STANDARD\"}]}" }
Ok([StateMachineListItem { state_machine_arn: Some("arn:aws:states:us-east-1:XXXXXXXXXXXX:stateMachine:HelloWorld"), name: Some("HelloWorld"), r#type: Some(Standard), creation_date: Some(DateTime { seconds: 1654853592, subsecond_nanos: 247999906 }) }, StateMachineListItem { state_machine_arn: Some("arn:aws:states:us-east-1:XXXXXXXXXXXX:stateMachine:MyStateMachine"), name: Some("MyStateMachine"), r#type: Some(Standard), creation_date: Some(DateTime { seconds: 1655372373, subsecond_nanos: 460000038 }) }])

Which works.

Then I try:

    let r = client
        .list_state_machines()
        .max_results(1)
        .send()
        .await?;
    println!("{:?}", r);
    let next_token = r.next_token.unwrap();

    let r = client
        .list_state_machines()
        .max_results(1)
        .next_token(next_token)
        .send()
        .await?;
    println!("{:?}", r);

which gives:

2022-06-17T06:49:59.128196Z TRACE send_operation{operation="ListStateMachines" service="sfn"}: aws_smithy_http_tower::dispatch: request=Request { method: POST, uri: https://states.us-east-1.amazonaws.com/, version: HTTP/1.1, headers: {"content-type": "application/x-amz-json-1.0", "x-amz-target": "AWSStepFunctions.ListStateMachines", "content-length": "16", "user-agent": "aws-sdk-rust/0.13.0 os/macos lang/rust/1.63.0-nightly", "x-amz-user-agent": "aws-sdk-rust/0.13.0 api/sfn/0.13.0 os/macos lang/rust/1.63.0-nightly", "x-amz-date": "20220617T064959Z", "authorization": Sensitive}, body: SdkBody { inner: Once(Some(b"{\"maxResults\":1}")), retryable: true } }
2022-06-17T06:49:59.364747Z TRACE send_operation{operation="ListStateMachines" service="sfn"}:load_response: aws_smithy_http::middleware: http_response=Response { status: 200, version: HTTP/1.1, headers: {"x-amzn-requestid": "313ff665-01f5-49f4-8ce2-ccc5a3abb400", "date": "Fri, 17 Jun 2022 06:49:59 GMT", "content-type": "application/x-amz-json-1.0", "content-length": "424"}, body: b"{\"nextToken\":\"AAAAKgAAAAIAAAAAAAAAATn/xpjCNn2XIh6yC9ovIKSejcld2qfWFaT5J/NxPEvKIBWNOsnNNbJGNPped3aKQAvljqFehwot4JEmHv3z50E2nz2iCope/PAd57JXQpcOpZapA23QRO4u1ColxFKtnp+ysiT6EwH+ccIBmdaL0Vb4gXCO72ZGzTLQgUrD+oUGAj24sUgpkj126Kj/cEnvl1ynUYwXSKnVuwxAxV98sgo=\",\"stateMachines\":[{\"creationDate\":1.654853592248E9,\"name\":\"HelloWorld\",\"stateMachineArn\":\"arn:aws:states:us-east-1:XXXXXXXXXXXX:stateMachine:HelloWorld\",\"type\":\"STANDARD\"}]}" }
ListStateMachinesOutput { state_machines: Some([StateMachineListItem { state_machine_arn: Some("arn:aws:states:us-east-1:XXXXXXXXXXXX:stateMachine:HelloWorld"), name: Some("HelloWorld"), r#type: Some(Standard), creation_date: Some(DateTime { seconds: 1654853592, subsecond_nanos: 247999906 }) }]), next_token: Some("AAAAKgAAAAIAAAAAAAAAATn/xpjCNn2XIh6yC9ovIKSejcld2qfWFaT5J/NxPEvKIBWNOsnNNbJGNPped3aKQAvljqFehwot4JEmHv3z50E2nz2iCope/PAd57JXQpcOpZapA23QRO4u1ColxFKtnp+ysiT6EwH+ccIBmdaL0Vb4gXCO72ZGzTLQgUrD+oUGAj24sUgpkj126Kj/cEnvl1ynUYwXSKnVuwxAxV98sgo=") }
2022-06-17T06:49:59.367867Z TRACE send_operation{operation="ListStateMachines" service="sfn"}: aws_smithy_http_tower::dispatch: request=Request { method: POST, uri: https://states.us-east-1.amazonaws.com/, version: HTTP/1.1, headers: {"content-type": "application/x-amz-json-1.0", "x-amz-target": "AWSStepFunctions.ListStateMachines", "content-length": "267", "user-agent": "aws-sdk-rust/0.13.0 os/macos lang/rust/1.63.0-nightly", "x-amz-user-agent": "aws-sdk-rust/0.13.0 api/sfn/0.13.0 os/macos lang/rust/1.63.0-nightly", "x-amz-date": "20220617T064959Z", "authorization": Sensitive}, body: SdkBody { inner: Once(Some(b"{\"maxResults\":1,\"nextToken\":\"AAAAKgAAAAIAAAAAAAAAATn/xpjCNn2XIh6yC9ovIKSejcld2qfWFaT5J/NxPEvKIBWNOsnNNbJGNPped3aKQAvljqFehwot4JEmHv3z50E2nz2iCope/PAd57JXQpcOpZapA23QRO4u1ColxFKtnp+ysiT6EwH+ccIBmdaL0Vb4gXCO72ZGzTLQgUrD+oUGAj24sUgpkj126Kj/cEnvl1ynUYwXSKnVuwxAxV98sgo=\"}")), retryable: true } }
2022-06-17T06:49:59.600578Z TRACE send_operation{operation="ListStateMachines" service="sfn"}:load_response: aws_smithy_http::middleware: http_response=Response { status: 200, version: HTTP/1.1, headers: {"x-amzn-requestid": "d1efc475-874a-44f6-9660-d6cea43d17cd", "date": "Fri, 17 Jun 2022 06:49:59 GMT", "content-type": "application/x-amz-json-1.0", "content-length": "180"}, body: b"{\"stateMachines\":[{\"creationDate\":1.65537237346E9,\"name\":\"MyStateMachine\",\"stateMachineArn\":\"arn:aws:states:us-east-1:XXXXXXXXXXXX:stateMachine:MyStateMachine\",\"type\":\"STANDARD\"}]}" }
ListStateMachinesOutput { state_machines: Some([StateMachineListItem { state_machine_arn: Some("arn:aws:states:us-east-1:XXXXXXXXXXXX:stateMachine:MyStateMachine"), name: Some("MyStateMachine"), r#type: Some(Standard), creation_date: Some(DateTime { seconds: 1655372373, subsecond_nanos: 460000038 }) }]), next_token:None }

Which works fine as well. I must have made an error in the handling of nextToken in my original post. This works fine - even with the strange looking nextTokens.

[github-actions[bot]]

Hello! Reopening this discussion to make it searchable.