diff --git a/src/main/java/com/ioa/IoASystem.java b/src/main/java/com/ioa/IoASystem.java index 746501a..86b64de 100644 --- a/src/main/java/com/ioa/IoASystem.java +++ b/src/main/java/com/ioa/IoASystem.java @@ -62,11 +62,11 @@ public class IoASystem { public static void main(String[] args) { var context = SpringApplication.run(IoASystem.class, args); - + AgentRegistry agentRegistry = context.getBean(AgentRegistry.class); TeamFormation teamFormation = context.getBean(TeamFormation.class); TaskManager taskManager = context.getBean(TaskManager.class); - + // Register some example agents AgentInfo agent1 = new AgentInfo("agent1", "General Assistant", Arrays.asList("general", "search"), @@ -77,23 +77,28 @@ public class IoASystem { agentRegistry.registerAgent(agent1.getId(), agent1); agentRegistry.registerAgent(agent2.getId(), agent2); - + // Create a sample task Task task = new Task("task1", "Plan a weekend trip to Paris", Arrays.asList("travel", "booking"), Arrays.asList("bookTravel", "findRestaurants", "getWeather")); - + // Form a team for the task List team = teamFormation.formTeam(task); System.out.println("Formed team: " + team); - - // Assign the task to the first agent in the team (simplified) + + if (team.isEmpty()) { + System.out.println("No suitable agents found for the task. Exiting."); + return; + } + + // Assign the task to the first agent in the team task.setAssignedAgent(team.get(0)); - + // Execute the task taskManager.addTask(task); taskManager.executeTask(task.getId()); - + // Print the result System.out.println("Task result: " + task.getResult()); } diff --git a/src/main/java/com/ioa/conversation/ConversationFSM.java b/src/main/java/com/ioa/conversation/ConversationFSM.java index 893871c..2bf96ac 100644 --- a/src/main/java/com/ioa/conversation/ConversationFSM.java +++ b/src/main/java/com/ioa/conversation/ConversationFSM.java @@ -22,11 +22,11 @@ public class ConversationFSM { String stateTransitionTask = "Decide the next conversation state based on this message: " + message.getContent() + "\nCurrent state: " + currentState; - String reasoning = model.generate(stateTransitionTask); + String reasoning = model.generate(stateTransitionTask, null); String decisionPrompt = "Based on this reasoning:\n" + reasoning + "\nProvide the next conversation state (DISCUSSION, TASK_ASSIGNMENT, EXECUTION, or CONCLUSION)."; - String response = model.generate(decisionPrompt); + String response = model.generate(decisionPrompt, null); ConversationState newState = ConversationState.valueOf(response.trim()); transitionTo(newState); diff --git a/src/main/java/com/ioa/model/BedrockLanguageModel.java b/src/main/java/com/ioa/model/BedrockLanguageModel.java index 3b42c71..5d68a26 100644 --- a/src/main/java/com/ioa/model/BedrockLanguageModel.java +++ b/src/main/java/com/ioa/model/BedrockLanguageModel.java @@ -12,6 +12,10 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import org.springframework.stereotype.Component; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Base64; + @Component public class BedrockLanguageModel { private final BedrockRuntimeClient bedrockClient; @@ -27,19 +31,35 @@ public class BedrockLanguageModel { this.modelId = modelId; } - public String generate(String prompt) { + public String generate(String prompt, String imagePath) { try { ObjectNode requestBody = objectMapper.createObjectNode(); requestBody.put("anthropic_version", "bedrock-2023-05-31"); ArrayNode messages = requestBody.putArray("messages"); ObjectNode message = messages.addObject(); message.put("role", "user"); - message.put("content", prompt); - requestBody.put("max_tokens", 500); requestBody.put("temperature", 0.7); requestBody.put("top_p", 0.9); + ArrayNode content = message.putArray("content"); + + if (imagePath != null && !imagePath.isEmpty()) { + byte[] imageBytes = Files.readAllBytes(Paths.get(imagePath)); + String base64Image = Base64.getEncoder().encodeToString(imageBytes); + + ObjectNode imageNode = content.addObject(); + imageNode.put("type", "image"); // Add type field + ObjectNode imageContent = imageNode.putObject("image"); + imageContent.put("format", "png"); + ObjectNode source = imageContent.putObject("source"); + source.put("bytes", base64Image); + } + + ObjectNode textNode = content.addObject(); + textNode.put("type", "text"); // Add type field + textNode.put("text", prompt); + String jsonPayload = objectMapper.writeValueAsString(requestBody); InvokeModelRequest invokeRequest = InvokeModelRequest.builder() @@ -53,7 +73,7 @@ public class BedrockLanguageModel { String responseBody = response.body().asUtf8String(); ObjectNode responseJson = (ObjectNode) objectMapper.readTree(responseBody); - return responseJson.path("content").asText(); + return responseJson.path("content").get(0).path("text").asText(); } catch (Exception e) { throw new RuntimeException("Error generating text with Bedrock", e); diff --git a/src/main/java/com/ioa/task/TaskManager.java b/src/main/java/com/ioa/task/TaskManager.java index 33169ac..664c0e4 100644 --- a/src/main/java/com/ioa/task/TaskManager.java +++ b/src/main/java/com/ioa/task/TaskManager.java @@ -41,13 +41,13 @@ public class TaskManager { "\nAssigned agent capabilities: " + agent.getCapabilities() + "\nAvailable tools: " + agent.getTools(); - String reasoning = model.generate(executionPlanningTask); + String reasoning = model.generate(executionPlanningTask, null); updateTaskProgress(taskId, "IN_PROGRESS", 50); String executionPrompt = "Based on this execution plan:\n" + reasoning + "\nExecute the task using the available tools and provide the result."; - String response = model.generate(executionPrompt); + String response = model.generate(executionPrompt, null); String result = executeToolsFromResponse(response, agent); diff --git a/src/main/java/com/ioa/team/TeamFormation.java b/src/main/java/com/ioa/team/TeamFormation.java index 0da1e67..c60a8dd 100644 --- a/src/main/java/com/ioa/team/TeamFormation.java +++ b/src/main/java/com/ioa/team/TeamFormation.java @@ -25,29 +25,38 @@ public class TeamFormation { List requiredTools = task.getRequiredTools(); List potentialAgents = agentRegistry.searchAgents(requiredCapabilities); + System.out.println("Potential agents: " + potentialAgents); + String teamFormationTask = "Form the best team for this task: " + task.getDescription() + + "\nRequired capabilities: " + requiredCapabilities + "\nRequired tools: " + requiredTools + - "\nAvailable agents and their tools: " + formatAgentTools(potentialAgents); + "\nAvailable agents and their tools: " + formatAgentTools(potentialAgents) + + "\nPlease respond with a comma-separated list of agent IDs that form the best team for this task."; - String reasoning = model.generate(teamFormationTask); - - String finalDecisionPrompt = "Based on this reasoning:\n" + reasoning + - "\nProvide the final team composition as a comma-separated list of agent IDs."; - String response = model.generate(finalDecisionPrompt); + System.out.println("Sending prompt to language model: " + teamFormationTask); + + String response = model.generate(teamFormationTask, null); + System.out.println("Language model response: " + response); + return parseTeamComposition(response, potentialAgents); } private String formatAgentTools(List agents) { return agents.stream() - .map(agent -> agent.getId() + ": " + agent.getTools()) + .map(agent -> agent.getId() + " (capabilities: " + agent.getCapabilities() + ", tools: " + agent.getTools() + ")") .collect(Collectors.joining(", ")); } private List parseTeamComposition(String composition, List potentialAgents) { List selectedIds = Arrays.asList(composition.split(",")); - return potentialAgents.stream() - .filter(agent -> selectedIds.contains(agent.getId())) + System.out.println("Parsed agent IDs: " + selectedIds); + + List team = potentialAgents.stream() + .filter(agent -> selectedIds.contains(agent.getId().trim())) .collect(Collectors.toList()); + + System.out.println("Final team: " + team); + return team; } } \ No newline at end of file diff --git a/src/main/java/com/ioa/util/TreeOfThought.java b/src/main/java/com/ioa/util/TreeOfThought.java index fbbecb5..60facdc 100644 --- a/src/main/java/com/ioa/util/TreeOfThought.java +++ b/src/main/java/com/ioa/util/TreeOfThought.java @@ -22,7 +22,7 @@ public class TreeOfThought { for (int i = 0; i < branches; i++) { String branchPrompt = "Consider the task: " + task + "\nCurrent path: " + path + "\nExplore a new branch of thought (branch " + (i+1) + "/" + branches + "):"; - String thought = model.generate(branchPrompt); + String thought = model.generate(branchPrompt, null); result.append("Branch ").append(i + 1).append(":\n"); result.append(thought).append("\n"); @@ -33,7 +33,7 @@ public class TreeOfThought { private String evaluateLeaf(String task, String path) { String prompt = "Evaluate the effectiveness of this approach for the task: " + task + "\nPath: " + path; - return model.generate(prompt); + return model.generate(prompt, null); } public BedrockLanguageModel getModel() { diff --git a/target/classes/com/ioa/IoASystem.class b/target/classes/com/ioa/IoASystem.class index 31d5438..9d479b3 100644 Binary files a/target/classes/com/ioa/IoASystem.class and b/target/classes/com/ioa/IoASystem.class differ diff --git a/target/classes/com/ioa/conversation/ConversationFSM.class b/target/classes/com/ioa/conversation/ConversationFSM.class index 1a51bbd..9f915b6 100644 Binary files a/target/classes/com/ioa/conversation/ConversationFSM.class and b/target/classes/com/ioa/conversation/ConversationFSM.class differ diff --git a/target/classes/com/ioa/model/BedrockLanguageModel.class b/target/classes/com/ioa/model/BedrockLanguageModel.class index 6077460..2570da7 100644 Binary files a/target/classes/com/ioa/model/BedrockLanguageModel.class and b/target/classes/com/ioa/model/BedrockLanguageModel.class differ diff --git a/target/classes/com/ioa/task/TaskManager.class b/target/classes/com/ioa/task/TaskManager.class index b56b9c9..0e7c460 100644 Binary files a/target/classes/com/ioa/task/TaskManager.class and b/target/classes/com/ioa/task/TaskManager.class differ diff --git a/target/classes/com/ioa/team/TeamFormation.class b/target/classes/com/ioa/team/TeamFormation.class index 8bf3a4b..ebf858b 100644 Binary files a/target/classes/com/ioa/team/TeamFormation.class and b/target/classes/com/ioa/team/TeamFormation.class differ diff --git a/target/classes/com/ioa/util/TreeOfThought.class b/target/classes/com/ioa/util/TreeOfThought.class index 69f5f4c..626813c 100644 Binary files a/target/classes/com/ioa/util/TreeOfThought.class and b/target/classes/com/ioa/util/TreeOfThought.class differ