Views
No views yet
1from transformers import AutoModelForCausalLM, AutoTokenizer
2
3tokenizer = AutoTokenizer.from_pretrained("Rel8ed/cleantech-cls")
4model = AutoModelForCausalLM.from_pretrained("Rel8ed/cleantech-cls")
5
6input_prompt = "[METAKEYWORD] innovation, technology, clean energy [TITLE] innovative clean energy solutions [META]" \
7 "leading provider of clean energy solutions. [ABOUT] we are committed to reducing environmental impact through" \
8 "cutting-edge clean energy solutions. [HOME] welcome to our website where we explore innovative technologies for a sustainable future."
9
10inputs = tokenizer.encode(input_prompt, return_tensors='pt')
11output = model.generate(inputs, max_length=50, num_return_sequences=5)
12
13print("Generated text:")
14for i, output in enumerate(outputs):
15 print(f"{i+1}: {tokenizer.decode(output, skip_special_tokens=True)}")1import re
2
3def normalize(s, truncate=100):
4 # Replace "\n" with " "
5 s = s.replace("\n", " ")
6
7 # Keep only letters (including accented letters) and spaces
8 s = re.sub(r"[^a-zA-Zà-üÀ-Ü ]", "", s)
9
10 # Split the string into words, truncate to the first 100 words, and join back into a string
11 words = s.split()
12 truncated = words[:truncate]
13 s = " ".join(truncated)
14
15 # Remove additional spaces
16 s = re.sub(r"\s+", " ", s)
17
18 return s
19
20
21
22def create_full_text(homepageText,metakeywords = "", title = "", meta = "", aboutText = "", truncate_limit=100):
23 return (
24 "[METAKEYWORD] " + normalize(metakeywords, truncate=truncate_limit) +
25 " [TITLE] " + normalize(title, truncate=truncate_limit) +
26 " [META] " + normalize(meta, truncate=truncate_limit) +
27 " [ABOUT] " + normalize(aboutText, truncate=truncate_limit) +
28 # Assuming we want to normalize homepageText with a much higher limit or no truncation
29 " [HOME] " + normalize(homepageText, truncate=truncate_limit)
30 ).strip()
31
32# Sample raw inputs
33metakeywords = "Green Energy, Sustainability"
34meta = "Exploring innovative solutions for a sustainable future."
35homepageText = "Welcome to our green energy platform where we share insights and innovations..."
36aboutText = "We are committed to advancing green energy solutions through research and development."
37title = "Green Energy Innovations"
38
39# Applying your preprocessing steps
40full_text = create_full_text(metakeywords, title, meta, aboutText, homepageText)
41
42print(full_text)1from transformers import pipeline
2import re
3
4model_name_or_path = "Rel8ed/cleantech-cls"
5
6classifier = pipeline('text-classification', model=model_name_or_path, max_length=512)
7
8def normalize(s, truncate=100):
9 s = s.replace("\n", " ")
10 s = re.sub(r"[^a-zA-Zà-üÀ-Ü ]", "", s)
11 words = s.split()
12 truncated = words[:truncate]
13 s = " ".join(truncated)
14 s = re.sub(r"\s+", " ", s)
15 return s
16
17
18def create_full_text(homepageText,metakeywords = "", title = "", meta = "", aboutText = "", truncate_limit=100):
19 return (
20 "[METAKEYWORD] " + normalize(metakeywords, truncate=truncate_limit) +
21 " [TITLE] " + normalize(title, truncate=truncate_limit) +
22 " [META] " + normalize(meta, truncate=truncate_limit) +
23 " [ABOUT] " + normalize(aboutText, truncate=truncate_limit) +
24 # Assuming we want to normalize homepageText with a much higher limit or no truncation
25 " [HOME] " + normalize(homepageText, truncate=truncate_limit)
26 ).strip()
27
28text = "Welcome to our green energy platform where we share insights and innovations"
29
30predictions = classifier(create_full_text(text))
31