Log a warning when the controller clients receive an error (#2499)

linkerd/app/inbound/src/policy/api.rs

@@ -109,7 +109,11 @@ impl Recover<tonic::Status> for GrpcRecover {
             return Err(status);
         }
 
-        tracing::trace!(%status, "Recovering");
+        tracing::warn!(
+            grpc.status = %status.code(),
+            grpc.message = status.message(),
+            "Unexpected policy controller response; retrying with a backoff",
+        );
         Ok(self.0.stream())
     }
 }

linkerd/app/outbound/src/policy/api.rs

@@ -113,8 +113,12 @@ impl Recover<tonic::Status> for GrpcRecover {
             tonic::Code::InvalidArgument | tonic::Code::FailedPrecondition => Err(status),
             // Indicates no policy for this target
             tonic::Code::NotFound | tonic::Code::Unimplemented => Err(status),
-            _ => {
-                tracing::debug!(%status, "Recovering");
+            code => {
+                tracing::warn!(
+                    grpc.status = %code,
+                    grpc.message = status.message(),
+                    "Unexpected policy controller response; retrying with a backoff",
+                );
                 Ok(self.0.stream())
             }
         }

linkerd/app/src/dst.rs

@@ -94,7 +94,11 @@ impl Recover<tonic::Status> for BackoffUnlessInvalidArgument {
             return Err(status);
         }
 
-        tracing::trace!(%status, "Recovering");
+        tracing::warn!(
+            grpc.status = %status.code(),
+            grpc.message = status.message(),
+            "Unexpected destination controller response; retrying with a backoff",
+        );
         Ok(self.0.stream())
     }
 }