Fully working Claude 3.5 on Bedrock code that uses agents

src/main/java/com/ioa/IoASystem.java

@@ -87,7 +87,12 @@ public class IoASystem {
         List<AgentInfo> team = teamFormation.formTeam(task);
         System.out.println("Formed team: " + team);
     
-        // Assign the task to the first agent in the team (simplified)
+        if (team.isEmpty()) {
+            System.out.println("No suitable agents found for the task. Exiting.");
+            return;
+        }
+    
+        // Assign the task to the first agent in the team
         task.setAssignedAgent(team.get(0));
     
         // Execute the task

src/main/java/com/ioa/conversation/ConversationFSM.java

@@ -22,11 +22,11 @@ public class ConversationFSM {
         String stateTransitionTask = "Decide the next conversation state based on this message: " + message.getContent() + 
                                      "\nCurrent state: " + currentState;
         
-        String reasoning = model.generate(stateTransitionTask);
+        String reasoning = model.generate(stateTransitionTask, null);
         
         String decisionPrompt = "Based on this reasoning:\n" + reasoning + 
                                 "\nProvide the next conversation state (DISCUSSION, TASK_ASSIGNMENT, EXECUTION, or CONCLUSION).";
-        String response = model.generate(decisionPrompt);
+        String response = model.generate(decisionPrompt, null);
         
         ConversationState newState = ConversationState.valueOf(response.trim());
         transitionTo(newState);

src/main/java/com/ioa/model/BedrockLanguageModel.java

@@ -12,6 +12,10 @@ import com.fasterxml.jackson.databind.node.ArrayNode;
 
 import org.springframework.stereotype.Component;
 
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Base64;
+
 @Component
 public class BedrockLanguageModel {
     private final BedrockRuntimeClient bedrockClient;
@@ -27,19 +31,35 @@ public class BedrockLanguageModel {
         this.modelId = modelId;
     }
 
-    public String generate(String prompt) {
+    public String generate(String prompt, String imagePath) {
         try {
             ObjectNode requestBody = objectMapper.createObjectNode();
             requestBody.put("anthropic_version", "bedrock-2023-05-31");
             ArrayNode messages = requestBody.putArray("messages");
             ObjectNode message = messages.addObject();
             message.put("role", "user");
-            message.put("content", prompt);
-            
             requestBody.put("max_tokens", 500);
             requestBody.put("temperature", 0.7);
             requestBody.put("top_p", 0.9);
 
+            ArrayNode content = message.putArray("content");
+
+            if (imagePath != null && !imagePath.isEmpty()) {
+                byte[] imageBytes = Files.readAllBytes(Paths.get(imagePath));
+                String base64Image = Base64.getEncoder().encodeToString(imageBytes);
+
+                ObjectNode imageNode = content.addObject();
+                imageNode.put("type", "image"); // Add type field
+                ObjectNode imageContent = imageNode.putObject("image");
+                imageContent.put("format", "png");
+                ObjectNode source = imageContent.putObject("source");
+                source.put("bytes", base64Image);
+            }
+
+            ObjectNode textNode = content.addObject();
+            textNode.put("type", "text"); // Add type field
+            textNode.put("text", prompt);
+
             String jsonPayload = objectMapper.writeValueAsString(requestBody);
 
             InvokeModelRequest invokeRequest = InvokeModelRequest.builder()
@@ -53,7 +73,7 @@ public class BedrockLanguageModel {
             String responseBody = response.body().asUtf8String();
 
             ObjectNode responseJson = (ObjectNode) objectMapper.readTree(responseBody);
-            return responseJson.path("content").asText();
+            return responseJson.path("content").get(0).path("text").asText();
 
         } catch (Exception e) {
             throw new RuntimeException("Error generating text with Bedrock", e);

src/main/java/com/ioa/task/TaskManager.java

@@ -41,13 +41,13 @@ public class TaskManager {
                                        "\nAssigned agent capabilities: " + agent.getCapabilities() +
                                        "\nAvailable tools: " + agent.getTools();
         
-        String reasoning = model.generate(executionPlanningTask);
+        String reasoning = model.generate(executionPlanningTask, null);
         
         updateTaskProgress(taskId, "IN_PROGRESS", 50);
         
         String executionPrompt = "Based on this execution plan:\n" + reasoning + 
                                  "\nExecute the task using the available tools and provide the result.";
-        String response = model.generate(executionPrompt);
+        String response = model.generate(executionPrompt, null);
         
         String result = executeToolsFromResponse(response, agent);
         

src/main/java/com/ioa/team/TeamFormation.java

@@ -25,29 +25,38 @@ public class TeamFormation {
         List<String> requiredTools = task.getRequiredTools();
         List<AgentInfo> potentialAgents = agentRegistry.searchAgents(requiredCapabilities);
         
+        System.out.println("Potential agents: " + potentialAgents);
+
         String teamFormationTask = "Form the best team for this task: " + task.getDescription() + 
+                                   "\nRequired capabilities: " + requiredCapabilities +
                                    "\nRequired tools: " + requiredTools +
-                                   "\nAvailable agents and their tools: " + formatAgentTools(potentialAgents);
+                                   "\nAvailable agents and their tools: " + formatAgentTools(potentialAgents) +
+                                   "\nPlease respond with a comma-separated list of agent IDs that form the best team for this task.";
         
-        String reasoning = model.generate(teamFormationTask);
+        System.out.println("Sending prompt to language model: " + teamFormationTask);
 
-        String finalDecisionPrompt = "Based on this reasoning:\n" + reasoning + 
-                                     "\nProvide the final team composition as a comma-separated list of agent IDs.";
-        String response = model.generate(finalDecisionPrompt);
+        String response = model.generate(teamFormationTask, null);
+        
+        System.out.println("Language model response: " + response);
 
         return parseTeamComposition(response, potentialAgents);
     }
 
     private String formatAgentTools(List<AgentInfo> agents) {
         return agents.stream()
-                .map(agent -> agent.getId() + ": " + agent.getTools())
+                .map(agent -> agent.getId() + " (capabilities: " + agent.getCapabilities() + ", tools: " + agent.getTools() + ")")
                 .collect(Collectors.joining(", "));
     }
 
     private List<AgentInfo> parseTeamComposition(String composition, List<AgentInfo> potentialAgents) {
         List<String> selectedIds = Arrays.asList(composition.split(","));
-        return potentialAgents.stream()
-                .filter(agent -> selectedIds.contains(agent.getId()))
+        System.out.println("Parsed agent IDs: " + selectedIds);
+
+        List<AgentInfo> team = potentialAgents.stream()
+                .filter(agent -> selectedIds.contains(agent.getId().trim()))
                 .collect(Collectors.toList());
+
+        System.out.println("Final team: " + team);
+        return team;
     }
 }

src/main/java/com/ioa/util/TreeOfThought.java

@@ -22,7 +22,7 @@ public class TreeOfThought {
         for (int i = 0; i < branches; i++) {
             String branchPrompt = "Consider the task: " + task + "\nCurrent path: " + path + 
                                   "\nExplore a new branch of thought (branch " + (i+1) + "/" + branches + "):";
-            String thought = model.generate(branchPrompt);
+            String thought = model.generate(branchPrompt, null);
             
             result.append("Branch ").append(i + 1).append(":\n");
             result.append(thought).append("\n");
@@ -33,7 +33,7 @@ public class TreeOfThought {
 
     private String evaluateLeaf(String task, String path) {
         String prompt = "Evaluate the effectiveness of this approach for the task: " + task + "\nPath: " + path;
-        return model.generate(prompt);
+        return model.generate(prompt, null);
     }
 
     public BedrockLanguageModel getModel() {