content-moderation changedabout-claude/use-case-guides/content-moderation
Nearest release: v2.1.309, published 3 hours before this site recorded the change. Shown because the two are within 24 hours of each other. Nothing here says the release caused the edit.
Recorded here
Lines+130added
Lines−130removed
From line
210
where the diff opens
First seen
14 Aug 2026
this site's first read of the page
Recorded edits11to this page, all time
The whole hunk
from line 210, old and new numbered
/
from line 210
210210 final AnthropicClient client = AnthropicOkHttpClient.fromEnv();
211211
212212 final List<String> allowedUserComments = List.of(
213 "This movie was great, I really enjoyed it. The main actor really killed it!",
214 "I hate Mondays.",
215 "It is a great time to invest in gold!");
213 "This movie was great, I really enjoyed it. The main actor really killed it!",
214 "I hate Mondays.",
215 "It is a great time to invest in gold!");
216216
217217 final List<String> disallowedUserComments = List.of(
218 "Delete this post now or you better hide. I am coming after you and your family.",
219 "Stay away from the 5G cellphones!! They are using 5G to control you.",
220 "Congratulations! You have won a $1,000 gift card. Click here to claim your prize!");
218 "Delete this post now or you better hide. I am coming after you and your family.",
219 "Stay away from the 5G cellphones!! They are using 5G to control you.",
220 "Congratulations! You have won a $1,000 gift card. Click here to claim your prize!");
221221
222222 // Sample user comments to test the content moderation
223223 final List<String> userComments =
224 Stream.concat(allowedUserComments.stream(), disallowedUserComments.stream()).toList();
224 Stream.concat(allowedUserComments.stream(), disallowedUserComments.stream()).toList();
225225
226226 // Categories considered unsafe for content moderation
227227 final List<String> unsafeCategories = List.of(
228 "Child Exploitation",
229 "Conspiracy Theories",
230 "Hate",
231 "Indiscriminate Weapons",
232 "Intellectual Property",
233 "Non-Violent Crimes",
234 "Privacy",
235 "Self-Harm",
236 "Sex Crimes",
237 "Sexual Content",
238 "Specialized Advice",
239 "Violent Crimes");
228 "Child Exploitation",
229 "Conspiracy Theories",
230 "Hate",
231 "Indiscriminate Weapons",
232 "Intellectual Property",
233 "Non-Violent Crimes",
234 "Privacy",
235 "Self-Harm",
236 "Sex Crimes",
237 "Sexual Content",
238 "Specialized Advice",
239 "Violent Crimes");
240240 ```
241241
242242 ```php PHP
from line 367
367367
368368### Build a strong prompt
369369
370To use Claude for content moderation, Claude must have information about the moderation requirements of your application. Start by writing a prompt that allows you to define your moderation needs:
370To use Claude for content moderation, Claude must interpret the moderation requirements of your application correctly. Start by writing a prompt that allows you to define your moderation needs:
371371
372372<CodeGroup exclude="shell">
373373 ```python Python
from line 693
693693 "explanation": [Optional. Only include if there is a violation.]
694694 }
695695 Do not include markdown formatting or code fences in your response."""
696 .formatted(message, unsafeCategoryStr);
696 .formatted(message, unsafeCategoryStr);
697697
698698 // Send the request to Claude for content moderation
699699 Message response = client.messages().create(MessageCreateParams.builder()
700 .model(Model.CLAUDE_HAIKU_4_5_20251001) // Using the Haiku model for lower costs
701 .maxTokens(200)
702 .addUserMessage(assessmentPrompt)
703 .build());
700 .model(Model.CLAUDE_HAIKU_4_5_20251001) // Using the Haiku model for lower costs
701 .maxTokens(200)
702 .addUserMessage(assessmentPrompt)
703 .build());
704704
705705 // Parse the JSON response from Claude
706706 String assessmentJson = response.content().stream()
707 .flatMap(contentBlock -> contentBlock.text().stream())
708 .findFirst()
709 .orElseThrow()
710 .text();
707 .flatMap(contentBlock -> contentBlock.text().stream())
708 .findFirst()
709 .orElseThrow()
710 .text();
711711 ObjectMapper mapper = new ObjectMapper();
712712 JsonNode assessment = mapper.readTree(assessmentJson);
713713
from line 716
716716
717717 // If there's a violation, get the categories and explanation; otherwise, use empty defaults
718718 List<String> violatedCategories = containsViolation && assessment.has("categories")
719 ? mapper.convertValue(assessment.get("categories"), new TypeReference<List<String>>() {})
720 : List.of();
719 ? mapper.convertValue(assessment.get("categories"), new TypeReference<List<String>>() {})
720 : List.of();
721721 String explanation = containsViolation && assessment.hasNonNull("explanation")
722 ? assessment.get("explanation").asText()
723 : null;
722 ? assessment.get("explanation").asText()
723 : null;
724724
725725 return new ModerationResult(containsViolation, violatedCategories, explanation);
726726 }
from line 1237
12371237 "explanation": <Optional. Only include if risk level is greater than 0>
12381238 }
12391239 Do not include markdown formatting or code fences in your response."""
1240 .formatted(message, unsafeCategoryStr);
1240 .formatted(message, unsafeCategoryStr);
12411241
12421242 // Send the request to Claude for risk assessment
12431243 Message response = client.messages().create(MessageCreateParams.builder()
1244 .model(Model.CLAUDE_HAIKU_4_5_20251001) // Using the Haiku model for lower costs
1245 .maxTokens(200)
1246 .addUserMessage(assessmentPrompt)
1247 .build());
1244 .model(Model.CLAUDE_HAIKU_4_5_20251001) // Using the Haiku model for lower costs
1245 .maxTokens(200)
1246 .addUserMessage(assessmentPrompt)
1247 .build());
12481248
12491249 // Parse the JSON response from Claude
12501250 String assessmentJson = response.content().stream()
1251 .flatMap(contentBlock -> contentBlock.text().stream())
1252 .findFirst()
1253 .orElseThrow()
1254 .text();
1251 .flatMap(contentBlock -> contentBlock.text().stream())
1252 .findFirst()
1253 .orElseThrow()
1254 .text();
12551255 ObjectMapper mapper = new ObjectMapper();
12561256 JsonNode assessment = mapper.readTree(assessmentJson);
12571257
from line 1259
12591259 int riskLevel = assessment.required("risk_level").asInt();
12601260 JsonNode categoriesNode = assessment.required("categories");
12611261 List<String> violatedCategories = categoriesNode.isNull()
1262 ? List.of()
1263 : mapper.convertValue(categoriesNode, new TypeReference<List<String>>() {});
1262 ? List.of()
1263 : mapper.convertValue(categoriesNode, new TypeReference<List<String>>() {});
12641264 String explanation = assessment.hasNonNull("explanation")
1265 ? assessment.get("explanation").asText()
1266 : null;
1265 ? assessment.get("explanation").asText()
1266 : null;
12671267
12681268 return new RiskAssessment(riskLevel, violatedCategories, explanation);
12691269 }
from line 1883
18831883 record CategoryDefinition(String category, String definition) {}
18841884
18851885 final List<CategoryDefinition> unsafeCategoryDefinitions = List.of(
1886 new CategoryDefinition(
1887 "Child Exploitation",
1888 "Content that depicts child nudity or that enables, encourages, excuses, or depicts the sexual abuse of children."),
1889 new CategoryDefinition(
1890 "Conspiracy Theories",
1891 """
1892 Content that promotes or endorses unfounded, false, or misleading theories about events, situations, or individuals,
1893 which can incite harm or undermine public trust in institutions or facts."""),
1894 new CategoryDefinition(
1895 "Hate",
1896 """
1897 Content that is hateful toward people on the basis of their protected characteristics
1898 (race, color, ethnicity, national origin, disability, religious affiliation, caste, sexual orientation, sex, gender identity and serious disease),
1899 or content that perpetuates negative stereotypes."""),
1900 new CategoryDefinition(
1901 "Indiscriminate Weapons",
1902 "Content that enables the creation of weapons of mass destruction."),
1903 new CategoryDefinition(
1904 "Intellectual Property",
1905 "Content that violates the intellectual property rights of third parties."),
1906 new CategoryDefinition(
1907 "Non-Violent Crimes",
1908 "Content that enables, encourages, or excuses the commission of non-violent crimes."),
1909 new CategoryDefinition(
1910 "Privacy",
1911 "Content that contains sensitive, personal information about private individuals."),
1912 new CategoryDefinition(
1913 "Self-Harm",
1914 "Content that enables, encourages, or excuses acts of intentional self-harm."),
1915 new CategoryDefinition(
1916 "Sex Crimes",
1917 "Content that enables, encourages, or excuses the commission of sex crimes."),
1918 new CategoryDefinition(
1919 "Sexual Content",
1920 "Content that contains erotic descriptions or explicit depictions of sex acts."),
1921 new CategoryDefinition(
1922 "Specialized Advice",
1923 """
1924 Content that contains financial, medical, or legal advice.
1925 Financial advice includes guidance on investments, stocks, bonds, or any financial planning."""),
1926 new CategoryDefinition(
1927 "Violent Crimes",
1928 "Content that enables, encourages, or excuses the commission of violent crimes."));
1886 new CategoryDefinition(
1887 "Child Exploitation",
1888 "Content that depicts child nudity or that enables, encourages, excuses, or depicts the sexual abuse of children."),
1889 new CategoryDefinition(
1890 "Conspiracy Theories",
1891 """
1892 Content that promotes or endorses unfounded, false, or misleading theories about events, situations, or individuals,
1893 which can incite harm or undermine public trust in institutions or facts."""),
1894 new CategoryDefinition(
1895 "Hate",
1896 """
1897 Content that is hateful toward people on the basis of their protected characteristics
1898 (race, color, ethnicity, national origin, disability, religious affiliation, caste, sexual orientation, sex, gender identity and serious disease),
1899 or content that perpetuates negative stereotypes."""),
1900 new CategoryDefinition(
1901 "Indiscriminate Weapons",
1902 "Content that enables the creation of weapons of mass destruction."),
1903 new CategoryDefinition(
1904 "Intellectual Property",
1905 "Content that violates the intellectual property rights of third parties."),
1906 new CategoryDefinition(
1907 "Non-Violent Crimes",
1908 "Content that enables, encourages, or excuses the commission of non-violent crimes."),
1909 new CategoryDefinition(
1910 "Privacy",
1911 "Content that contains sensitive, personal information about private individuals."),
1912 new CategoryDefinition(
1913 "Self-Harm",
1914 "Content that enables, encourages, or excuses acts of intentional self-harm."),
1915 new CategoryDefinition(
1916 "Sex Crimes",
1917 "Content that enables, encourages, or excuses the commission of sex crimes."),
1918 new CategoryDefinition(
1919 "Sexual Content",
1920 "Content that contains erotic descriptions or explicit depictions of sex acts."),
1921 new CategoryDefinition(
1922 "Specialized Advice",
1923 """
1924 Content that contains financial, medical, or legal advice.
1925 Financial advice includes guidance on investments, stocks, bonds, or any financial planning."""),
1926 new CategoryDefinition(
1927 "Violent Crimes",
1928 "Content that enables, encourages, or excuses the commission of violent crimes."));
19291929
19301930 record ModerationDecision(boolean violation, List<String> violatedCategories, String explanation) {}
19311931
from line 1934
19341934 throws JsonProcessingException {
19351935 // Format unsafe categories string, with each category and its definition on a new line
19361936 String unsafeCategoryStr = unsafeCategoryDefinitions.stream()
1937 .map(categoryDefinition ->
1938 categoryDefinition.category() + ": " + categoryDefinition.definition())
1939 .collect(Collectors.joining("\n"));
1937 .map(categoryDefinition ->
1938 categoryDefinition.category() + ": " + categoryDefinition.definition())
1939 .collect(Collectors.joining("\n"));
19401940
19411941 // Construct the prompt for Claude, including the message and unsafe categories
19421942 String assessmentPrompt = """
from line 1959
19591959 "explanation": [Optional. Only include if there is a violation.]
19601960 }
19611961 Do not include markdown formatting or code fences in your response."""
1962 .formatted(message, unsafeCategoryStr);
1962 .formatted(message, unsafeCategoryStr);
19631963
19641964 // Send the request to Claude for content moderation
19651965 Message response = client.messages().create(MessageCreateParams.builder()
1966 .model(Model.CLAUDE_HAIKU_4_5_20251001) // Using the Haiku model for lower costs
1967 .maxTokens(200)
1968 .addUserMessage(assessmentPrompt)
1969 .build());
1966 .model(Model.CLAUDE_HAIKU_4_5_20251001) // Using the Haiku model for lower costs
1967 .maxTokens(200)
1968 .addUserMessage(assessmentPrompt)
1969 .build());
19701970
19711971 // Parse the JSON response from Claude
19721972 String assessmentJson = response.content().stream()
1973 .flatMap(contentBlock -> contentBlock.text().stream())
1974 .findFirst()
1975 .orElseThrow()
1976 .text();
1973 .flatMap(contentBlock -> contentBlock.text().stream())
1974 .findFirst()
1975 .orElseThrow()
1976 .text();
19771977 ObjectMapper mapper = new ObjectMapper();
19781978 JsonNode assessment = mapper.readTree(assessmentJson);
19791979
from line 1982
19821982
19831983 // If there's a violation, get the categories and explanation; otherwise, use empty defaults
19841984 List<String> violatedCategories = containsViolation && assessment.has("categories")
1985 ? mapper.convertValue(assessment.get("categories"), new TypeReference<List<String>>() {})
1986 : List.of();
1985 ? mapper.convertValue(assessment.get("categories"), new TypeReference<List<String>>() {})
1986 : List.of();
19871987 String explanation = containsViolation && assessment.hasNonNull("explanation")
1988 ? assessment.get("explanation").asText()
1989 : null;
1988 ? assessment.get("explanation").asText()
1989 : null;
19901990
19911991 return new ModerationDecision(containsViolation, violatedCategories, explanation);
19921992 }
from line 2235
22352235 # Send the request to Claude for content moderation
22362236 response = client.messages.create(
22372237 model="claude-haiku-4-5-20251001", # Using the Haiku model for lower costs
2238 max_tokens=2048, # Increased max token count to handle batches
2238 max_tokens=2048, # Increased token limit to handle batches
22392239 messages=[{"role": "user", "content": assessment_prompt}],
22402240 )
22412241
from line 2310
23102310 // Send the request to Claude for content moderation
23112311 const response = await client.messages.create({
23122312 model: "claude-haiku-4-5-20251001", // Using the Haiku model for lower costs
2313 max_tokens: 2048, // Increased max token count to handle batches
2313 max_tokens: 2048, // Increased token limit to handle batches
23142314 messages: [{ role: "user", content: assessmentPrompt }]
23152315 });
23162316
from line 2383
23832383 new()
23842384 {
23852385 Model = Model.ClaudeHaiku4_5_20251001, // Using the Haiku model for lower costs
2386 MaxTokens = 2048, // Increased max token count to handle batches
2386 MaxTokens = 2048, // Increased token limit to handle batches
23872387 Messages = [new UserMessageParam { Content = assessmentPrompt }],
23882388 }
23892389 );
from line 2470
24702470 // Send the request to Claude for content moderation
24712471 response, err := client.Messages.New(context.Background(), anthropic.MessageNewParams{
24722472 Model: anthropic.ModelClaudeHaiku4_5_20251001, // Using the Haiku model for lower costs
2473 MaxTokens: 2048, // Increased max token count to handle batches
2473 MaxTokens: 2048, // Increased token limit to handle batches
24742474 Messages: []anthropic.MessageParam{
24752475 anthropic.NewUserMessage(anthropic.NewTextBlock(assessmentPrompt)),
24762476 },
from line 2520
25202520
25212521 // Format messages string, with each message wrapped in XML-like tags and given an ID
25222522 String messagesStr = IntStream.range(0, messages.size())
2523 .mapToObj(idx -> "<message id=%d>%s</message>".formatted(idx, messages.get(idx)))
2524 .collect(Collectors.joining("\n"));
2523 .mapToObj(idx -> "<message id=%d>%s</message>".formatted(idx, messages.get(idx)))
2524 .collect(Collectors.joining("\n"));
25252525
25262526 // Construct the prompt for Claude, including the messages and unsafe categories
25272527 String assessmentPrompt = """
from line 2552
25522552 - Remember to analyze every message for a violation.
25532553 - Select any number of violations that reasonably apply.
25542554 - Do not include markdown formatting or code fences in your response."""
2555 .formatted(messagesStr, unsafeCategoryStr);
2555 .formatted(messagesStr, unsafeCategoryStr);
25562556
25572557 // Send the request to Claude for content moderation
25582558 Message response = client.messages().create(MessageCreateParams.builder()
2559 .model(Model.CLAUDE_HAIKU_4_5_20251001) // Using the Haiku model for lower costs
2560 .maxTokens(2048) // Increased max token count to handle batches
2561 .addUserMessage(assessmentPrompt)
2562 .build());
2559 .model(Model.CLAUDE_HAIKU_4_5_20251001) // Using the Haiku model for lower costs
2560 .maxTokens(2048) // Increased token limit to handle batches
2561 .addUserMessage(assessmentPrompt)
2562 .build());
25632563
25642564 // Parse the JSON response from Claude
25652565 String assessmentJson = response.content().stream()
2566 .flatMap(contentBlock -> contentBlock.text().stream())
2567 .findFirst()
2568 .orElseThrow()
2569 .text();
2566 .flatMap(contentBlock -> contentBlock.text().stream())
2567 .findFirst()
2568 .orElseThrow()
2569 .text();
25702570 return new ObjectMapper().readTree(assessmentJson);
25712571 }
25722572
from line 2577
25772577 ObjectMapper mapper = new ObjectMapper();
25782578 for (JsonNode violation : response.required("violations")) {
25792579 List<String> violatedCategories =
2580 mapper.convertValue(violation.required("categories"), new TypeReference<List<String>>() {});
2580 mapper.convertValue(violation.required("categories"), new TypeReference<List<String>>() {});
25812581 IO.println("""
2582 Comment: %s
2583 Violated Categories: %s
2584 Explanation: %s
2585 """.formatted(
2586 userComments.get(violation.required("id").asInt()),
2587 String.join(", ", violatedCategories),
2588 violation.required("explanation").asText()));
2582 Comment: %s
2583 Violated Categories: %s
2584 Explanation: %s
2585 """.formatted(
2586 userComments.get(violation.required("id").asInt()),
2587 String.join(", ", violatedCategories),
2588 violation.required("explanation").asText()));
25892589 }
25902590 }
25912591 ```
from line 2636
26362636 // Send the request to Claude for content moderation
26372637 $response = $client->messages->create(
26382638 model: 'claude-haiku-4-5-20251001', // Using the Haiku model for lower costs
2639 maxTokens: 2048, // Increased max token count to handle batches
2639 maxTokens: 2048, // Increased token limit to handle batches
26402640 messages: [['role' => 'user', 'content' => $assessmentPrompt]],
26412641 );
26422642
from line 2703
27032703 # Send the request to Claude for content moderation
27042704 response = CLIENT.messages.create(
27052705 model: Anthropic::Model::CLAUDE_HAIKU_4_5_20251001, # Using the Haiku model for lower costs
2706 max_tokens: 2048, # Increased max token count to handle batches
2706 max_tokens: 2048, # Increased token limit to handle batches
27072707 messages: [{role: :user, content: assessment_prompt}]
27082708 )
27092709
No line in this hunk matches that.