Views
No views yet
1import torch
2from transformers import AutoTokenizer, BertForTokenClassification
3import math
4
5model_path = "tim1900/bert-chunker-3"
6
7tokenizer = AutoTokenizer.from_pretrained(
8 model_path,
9 padding_side="right",
10 model_max_length=255,
11 trust_remote_code=True,
12)
13
14device = "cpu" # or 'cuda'
15
16model = BertForTokenClassification.from_pretrained(
17 model_path,
18).to(device)
19
20def chunk_text(model, text, tokenizer, prob_threshold=0.5):
21 # slide context window chunking
22 MAX_TOKENS = 255
23 tokens = tokenizer(text, return_tensors="pt", truncation=False)
24 input_ids = tokens["input_ids"]
25 attention_mask = tokens["attention_mask"][:, 0:MAX_TOKENS]
26 attention_mask = attention_mask.to(model.device)
27 CLS = input_ids[:, 0].unsqueeze(0)
28 SEP = input_ids[:, -1].unsqueeze(0)
29 input_ids = input_ids[:, 1:-1]
30 model.eval()
31 split_str_poses = []
32 token_pos = []
33 windows_start = 0
34 windows_end = 0
35 logits_threshold = math.log(1 / prob_threshold - 1)
36 print(f"Processing {input_ids.shape[1]} tokens...")
37 while windows_end <= input_ids.shape[1]:
38 windows_end = windows_start + MAX_TOKENS - 2
39
40 ids = torch.cat((CLS, input_ids[:, windows_start:windows_end], SEP), 1)
41
42 ids = ids.to(model.device)
43
44 output = model(
45 input_ids=ids,
46 attention_mask=torch.ones(1, ids.shape[1], device=model.device),
47 )
48 logits = output["logits"][:, 1:-1, :]
49 chunk_decision = logits[:, :, 1] > (logits[:, :, 0] - logits_threshold)
50 greater_rows_indices = torch.where(chunk_decision)[1].tolist()
51
52 # null or not
53 if len(greater_rows_indices) > 0 and (
54 not (greater_rows_indices[0] == 0 and len(greater_rows_indices) == 1)
55 ):
56
57 split_str_pos = [
58 tokens.token_to_chars(sp + windows_start + 1).start
59 for sp in greater_rows_indices
60 if sp > 0
61 ]
62 token_pos += [
63 sp + windows_start for sp in greater_rows_indices if sp > 0
64 ]
65 split_str_poses += split_str_pos
66
67 windows_start = greater_rows_indices[-1] + windows_start
68
69 else:
70
71 windows_start = windows_end
72
73 substrings = [
74 text[i:j] for i, j in zip([0] + split_str_poses, split_str_poses + [len(text)])
75 ]
76 token_pos = [0] + token_pos
77 return substrings, token_pos
78
79
80# chunking code docs
81print("\n>>>>>>>>> Chunking code docs...")
82doc = r"""
83Of course, as our first example shows, it is not always _necessary_ to declare an expression holder before it is created or used. But doing so provides an extra measure of clarity to models, so we strongly recommend it.
84
85## Chapter 4 The Basics
86
87## Chapter 5 The DCP Ruleset
88
89### 5.1 A taxonomy of curvature
90
91In disciplined convex programming, a scalar expression is classified by its _curvature_. There are four categories of curvature: _constant_, _affine_, _convex_, and _concave_. For a function \(f:\mathbf{R}^{n}\rightarrow\mathbf{R}\) defined on all \(\mathbf{R}^{n}\)the categories have the following meanings:
92
93\[\begin{array}{llll}\text{constant}&f(\alpha x+(1-\alpha)y)=f(x)&\forall x,y\in \mathbf{R}^{n},\;\alpha\in\mathbf{R}\\ \text{affine}&f(\alpha x+(1-\alpha)y)=\alpha f(x)+(1-\alpha)f(y)&\forall x,y\in \mathbf{R}^{n},\;\alpha\in\mathbf{R}\\ \text{convex}&f(\alpha x+(1-\alpha)y)\leq\alpha f(x)+(1-\alpha)f(y)&\forall x,y \in\mathbf{R}^{n},\;\alpha\in[0,1]\\ \text{concave}&f(\alpha x+(1-\alpha)y)\geq\alpha f(x)+(1-\alpha)f(y)&\forall x,y \in\mathbf{R}^{n},\;\alpha\in[0,1]\end{array}\]
94
95Of course, there is significant overlap in these categories. For example, constant expressions are also affine, and (real) affine expressions are both convex and concave.
96
97Convex and concave expressions are real by definition. Complex constant and affine expressions can be constructed, but their usage is more limited; for example, they cannot appear as the left- or right-hand side of an inequality constraint.
98
99### Top-level rules
100
101CVX supports three different types of disciplined convex programs:
102
103* A _minimization problem_, consisting of a convex objective function and zero or more constraints.
104* A _maximization problem_, consisting of a concave objective function and zero or more constraints.
105* A _feasibility problem_, consisting of one or more constraints and no objective.
106
107### Constraints
108
109Three types of constraints may be specified in disciplined convex programs:
110
111* An _equality constraint_, constructed using \(==\), where both sides are affine.
112* A _less-than inequality constraint_, using \(<=\), where the left side is convex and the right side is concave.
113* A _greater-than inequality constraint_, using \(>=\), where the left side is concave and the right side is convex.
114
115_Non_-equality constraints, constructed using \(\sim=\), are never allowed. (Such constraints are not convex.)
116
117One or both sides of an equality constraint may be complex; inequality constraints, on the other hand, must be real. A complex equality constraint is equivalent to two real equality constraints, one for the real part and one for the imaginary part. An equality constraint with a real side and a complex side has the effect of constraining the imaginary part of the complex side to be zero."""
118# Chunk the text. The prob_threshold should be between (0, 1). The lower it is, the more chunks will be generated.
119# Therefore adjust it to your need, when prob_threshold is small like 0.000001, each token is one chunk,
120# when it is set to 1, the whole text will be one chunk.
121chunks, token_pos = chunk_text(model, doc, tokenizer, prob_threshold=0.5)
122
123# print chunks
124for i, (c, t) in enumerate(zip(chunks, token_pos)):
125 print(f"-----chunk: {i}----token_idx: {t}--------")
126 print(c)
127
128
129# chunking ads
130print("\n>>>>>>>>> Chunking ads...")
131
132ad = r"""The causes and effects of dropouts in vocational and professional education are more pressing than ever. A decreasing attractiveness of vocational education, particularly in payment and quality, causes higher dropout rates while hitting ongoing demographic changes resulting in extensive skill shortages for many regions. Therefore, tackling the internationally high dropout rates is of utmost political and scientific interest. This thematic issue contributes to the conceptualization, analysis, and prevention of vocational and professional dropouts by bringing together current research that progresses to a deeper processual understanding and empirical modelling of dropouts. It aims to expand our understanding of how dropout and decision processes leading to dropout can be conceptualized and measured in vocational and professional contexts. Another aim is to gather empirical studies on both predictors and dropout consequences. Based on this knowledge, the thematic issue intends to provide evidence of effective interventions to avoid dropouts and identify promising ways for future dropout research in professional and vocational education to support evidence-based vocational education policy.
133
134We thus welcome research contributions (original empirical and conceptual/measurement-related articles, literature reviews, meta-analyses) on dropouts (e.g., premature terminations, intentions to terminate, vertical and horizontal dropouts) that are situated in vocational and professional education at workplaces, schools, or other tertiary professional education institutions.
135
136
137Part 1 of the thematic series outlines central theories and measurement concepts for vocational and professional dropouts. Part 2 outlines measurement approaches for dropout. Part 3 investigates relevant predictors of dropout. Part 4 analyzes the effects of dropout on an individual, organizational, and systemic level. Part 5 deals with programs and interventions for the prevention of dropouts.
138
139We welcome papers that include but are not limited to:
140
141Theoretical papers on the concept and processes of vocational and professional dropout or retention
142Measurement approaches to assess dropout or retention
143Quantitative and qualitative papers on the causes of dropout or retention
144Quantitative and qualitative papers on the effects of dropout or retention on learners, providers/organizations and the (educational) system
145Design-based research and experimental papers on dropout prevention programs or retention
146Submission instructions
147Before submitting your manuscript, please ensure you have carefully read the Instructions for Authors for Empirical Research in Vocational Education and Training. The complete manuscript should be submitted through the Empirical Research in Vocational Education and Training submission system. To ensure that you submit to the correct thematic series please select the appropriate section in the drop-down menu upon submission. In addition, indicate within your cover letter that you wish your manuscript to be considered as part of the thematic series on series title. All submissions will undergo rigorous peer review, and accepted articles will be published within the journal as a collection.
148
149Lead Guest Editor:
150Prof. Dr. Viola Deutscher, University of Mannheim
151viola.deutscher@uni-mannheim.de
152
153Guest Editors:
154Prof. Dr. Stefanie Findeisen, University of Konstanz
155stefanie.findeisen@uni-konstanz.de
156
157Prof. Dr. Christian Michaelis, Georg-August-University of Göttingen
158christian.michaelis@wiwi.uni-goettingen.de
159
160Deadline for submission
161This Call for Papers is open from now until 29 February 2023. Submitted papers will be reviewed in a timely manner and published directly after acceptance (i.e., without waiting for the accomplishment of all other contributions). Thanks to the Empirical Research in Vocational Education and Training (ERVET) open access policy, the articles published in this thematic issue will have a wide, global audience.
162
163Option of submitting abstracts: Interested authors should submit a letter of intent including a working title for the manuscript, names, affiliations, and contact information for all authors, and an abstract of no more than 500 words to the lead guest editor Viola Deutscher (viola.deutscher@uni-mannheim.de) by July, 31st 2023. Due to technical issues, we also ask authors who already submitted an abstract before May, 30th to send their abstracts again to the address stated above. However, abstract submission is optional and is not mandatory for the full paper submission.
164
165Different dropout directions in vocational education and training: the role of the initiating party and trainees’ reasons for dropping out
166The high rates of premature contract termination (PCT) in vocational education and training (VET) programs have led to an increasing number of studies examining the reasons why adolescents drop out. Since adol...
167
168Authors:Christian Michaelis and Stefanie Findeisen
169Citation:Empirical Research in Vocational Education and Training 2024 16:15
170Content type:Research
171Published on: 6 August 2024"
172"""
173# Chunk the text. The prob_threshold should be between (0, 1). The lower it is, the more chunks will be generated.
174# Therefore adjust it to your need, when prob_threshold is small like 0.000001, each token is one chunk,
175# when it is set to 1, the whole text will be one chunk.
176chunks, token_pos = chunk_text(model, ad, tokenizer, prob_threshold=0.5)
177
178# print chunks
179for i, (c, t) in enumerate(zip(chunks, token_pos)):
180 print(f"-----chunk: {i}----token_idx: {t}--------")
181 print(c)1import torch
2from transformers import AutoTokenizer, BertForTokenClassification
3import math
4
5model_path = "tim1900/bert-chunker-3"
6
7tokenizer = AutoTokenizer.from_pretrained(
8 model_path,
9 padding_side="right",
10 model_max_length=255,
11 trust_remote_code=True,
12)
13
14device = "cpu" # or 'cuda'
15
16model = BertForTokenClassification.from_pretrained(
17 model_path,
18).to(device)
19
20def chunk_text_with_max_chunk_size(model, text, tokenizer, prob_threshold=0.5,max_tokens_per_chunk = 400):
21 with torch.no_grad():
22
23 # slide context window chunking
24 MAX_TOKENS = 255
25 tokens = tokenizer(text, return_tensors="pt", truncation=False)
26 input_ids = tokens["input_ids"]
27 attention_mask = tokens["attention_mask"][:, 0:MAX_TOKENS]
28 attention_mask = attention_mask.to(model.device)
29 CLS = input_ids[:, 0].unsqueeze(0)
30 SEP = input_ids[:, -1].unsqueeze(0)
31 input_ids = input_ids[:, 1:-1]
32 model.eval()
33 split_str_poses = []
34 token_pos = []
35 windows_start = 0
36 windows_end = 0
37 logits_threshold = math.log(1 / prob_threshold - 1)
38
39 unchunk_tokens = 0
40 backup_pos = None
41 best_logits = torch.finfo(torch.float32).min
42 STEP = round(((MAX_TOKENS - 2)//2)*1.75 )
43 print(f"Processing {input_ids.shape[1]} tokens...")
44 # while windows_end <= input_ids.shape[1]:
45 while windows_start < input_ids.shape[1]:
46 windows_end = windows_start + MAX_TOKENS - 2
47 ids = torch.cat((CLS, input_ids[:, windows_start:windows_end], SEP), 1)
48 ids = ids.to(model.device)
49 output = model(
50 input_ids=ids,
51 attention_mask=torch.ones(1, ids.shape[1], device=model.device),
52 )
53 logits = output["logits"][:, 1:-1, :]
54
55
56 logit_diff = logits[:, :, 1] - logits[:, :, 0]
57
58
59 chunk_decision = logit_diff > - logits_threshold
60 greater_rows_indices = torch.where(chunk_decision)[1].tolist()
61
62 # null or not
63 if len(greater_rows_indices) > 0 and (
64 not (greater_rows_indices[0] == 0 and len(greater_rows_indices) == 1)
65 ):
66
67
68 unchunk_tokens_this_window = greater_rows_indices[0] if greater_rows_indices[0]!=0 else greater_rows_indices[1]#exclude the fist index
69
70 # manually chunk
71 if unchunk_tokens + unchunk_tokens_this_window > max_tokens_per_chunk: #change ">" to ">=" if buggy for the moment
72 big_windows_end = max_tokens_per_chunk - unchunk_tokens
73 max_value, max_index= logit_diff[:,1:big_windows_end].max(), logit_diff[:,1:big_windows_end].argmax() + 1
74 if best_logits < max_value:
75 backup_pos = windows_start + max_index
76
77 windows_start = backup_pos
78
79
80 split_str_pos = [tokens.token_to_chars(backup_pos + 1).start]
81 split_str_poses = split_str_poses + split_str_pos
82 token_pos = token_pos + [backup_pos]
83 best_logits = torch.finfo(torch.float32).min
84 backup_pos = -1
85 unchunk_tokens = 0
86
87 # auto chunk
88 else:
89
90 if len(greater_rows_indices) >= 2:
91 for gi, (gri0,gri1) in enumerate(zip(greater_rows_indices[:-1],greater_rows_indices[1:])):
92
93 if gri1 - gri0 > max_tokens_per_chunk:
94 greater_rows_indices=greater_rows_indices[:gi+1]
95 break
96
97 split_str_pos = [tokens.token_to_chars(sp + windows_start + 1).start for sp in greater_rows_indices if sp > 0]
98 split_str_poses = split_str_poses + split_str_pos
99 token_pos = token_pos+ [sp + windows_start for sp in greater_rows_indices if sp > 0]
100
101 windows_start = greater_rows_indices[-1] + windows_start
102 best_logits = torch.finfo(torch.float32).min
103 backup_pos = -1
104 unchunk_tokens = 0
105
106 else:
107
108 # unchunk_tokens_this_window = min(windows_end - windows_start,STEP)
109 unchunk_tokens_this_window = min(windows_start+STEP,input_ids.shape[1]) - windows_start
110
111 # manually chunk
112 if unchunk_tokens + unchunk_tokens_this_window > max_tokens_per_chunk: #change ">" to ">=" if buggy for the moment
113 big_windows_end = max_tokens_per_chunk - unchunk_tokens
114 if logit_diff.shape[1] > 1:
115
116 max_value, max_index= logit_diff[:,1:big_windows_end].max(), logit_diff[:,1:big_windows_end].argmax() + 1
117 if best_logits < max_value:
118 backup_pos = windows_start + max_index
119
120
121 windows_start = backup_pos
122 split_str_pos = [tokens.token_to_chars(backup_pos + 1).start]
123 split_str_poses = split_str_poses + split_str_pos
124 token_pos = token_pos + [backup_pos]
125 best_logits = torch.finfo(torch.float32).min
126 backup_pos = -1
127 unchunk_tokens = 0
128 else:
129 # auto leave
130 if logit_diff.shape[1] > 1:
131 max_value, max_index= logit_diff[:,1:].max(), logit_diff[:,1:].argmax() + 1
132 if best_logits < max_value:
133 best_logits = max_value
134 backup_pos = windows_start + max_index
135
136 unchunk_tokens = unchunk_tokens + STEP
137 windows_start = windows_start + STEP
138
139 substrings = [
140 text[i:j] for i, j in zip([0] + split_str_poses, split_str_poses + [len(text)])
141 ]
142 token_pos = [0] + token_pos
143 return substrings, token_pos
144
145# chunking ads
146print("\n>>>>>>>>> Chunking ads...")
147
148ad = r"""The causes and effects of dropouts in vocational and professional education are more pressing than ever. A decreasing attractiveness of vocational education, particularly in payment and quality, causes higher dropout rates while hitting ongoing demographic changes resulting in extensive skill shortages for many regions. Therefore, tackling the internationally high dropout rates is of utmost political and scientific interest. This thematic issue contributes to the conceptualization, analysis, and prevention of vocational and professional dropouts by bringing together current research that progresses to a deeper processual understanding and empirical modelling of dropouts. It aims to expand our understanding of how dropout and decision processes leading to dropout can be conceptualized and measured in vocational and professional contexts. Another aim is to gather empirical studies on both predictors and dropout consequences. Based on this knowledge, the thematic issue intends to provide evidence of effective interventions to avoid dropouts and identify promising ways for future dropout research in professional and vocational education to support evidence-based vocational education policy.
149
150We thus welcome research contributions (original empirical and conceptual/measurement-related articles, literature reviews, meta-analyses) on dropouts (e.g., premature terminations, intentions to terminate, vertical and horizontal dropouts) that are situated in vocational and professional education at workplaces, schools, or other tertiary professional education institutions.
151
152
153Part 1 of the thematic series outlines central theories and measurement concepts for vocational and professional dropouts. Part 2 outlines measurement approaches for dropout. Part 3 investigates relevant predictors of dropout. Part 4 analyzes the effects of dropout on an individual, organizational, and systemic level. Part 5 deals with programs and interventions for the prevention of dropouts.
154
155We welcome papers that include but are not limited to:
156
157Theoretical papers on the concept and processes of vocational and professional dropout or retention
158Measurement approaches to assess dropout or retention
159Quantitative and qualitative papers on the causes of dropout or retention
160Quantitative and qualitative papers on the effects of dropout or retention on learners, providers/organizations and the (educational) system
161Design-based research and experimental papers on dropout prevention programs or retention
162Submission instructions
163Before submitting your manuscript, please ensure you have carefully read the Instructions for Authors for Empirical Research in Vocational Education and Training. The complete manuscript should be submitted through the Empirical Research in Vocational Education and Training submission system. To ensure that you submit to the correct thematic series please select the appropriate section in the drop-down menu upon submission. In addition, indicate within your cover letter that you wish your manuscript to be considered as part of the thematic series on series title. All submissions will undergo rigorous peer review, and accepted articles will be published within the journal as a collection.
164
165Lead Guest Editor:
166Prof. Dr. Viola Deutscher, University of Mannheim
167viola.deutscher@uni-mannheim.de
168
169Guest Editors:
170Prof. Dr. Stefanie Findeisen, University of Konstanz
171stefanie.findeisen@uni-konstanz.de
172
173Prof. Dr. Christian Michaelis, Georg-August-University of Göttingen
174christian.michaelis@wiwi.uni-goettingen.de
175
176Deadline for submission
177This Call for Papers is open from now until 29 February 2023. Submitted papers will be reviewed in a timely manner and published directly after acceptance (i.e., without waiting for the accomplishment of all other contributions). Thanks to the Empirical Research in Vocational Education and Training (ERVET) open access policy, the articles published in this thematic issue will have a wide, global audience.
178
179Option of submitting abstracts: Interested authors should submit a letter of intent including a working title for the manuscript, names, affiliations, and contact information for all authors, and an abstract of no more than 500 words to the lead guest editor Viola Deutscher (viola.deutscher@uni-mannheim.de) by July, 31st 2023. Due to technical issues, we also ask authors who already submitted an abstract before May, 30th to send their abstracts again to the address stated above. However, abstract submission is optional and is not mandatory for the full paper submission.
180
181Different dropout directions in vocational education and training: the role of the initiating party and trainees’ reasons for dropping out
182The high rates of premature contract termination (PCT) in vocational education and training (VET) programs have led to an increasing number of studies examining the reasons why adolescents drop out. Since adol...
183
184Authors:Christian Michaelis and Stefanie Findeisen
185Citation:Empirical Research in Vocational Education and Training 2024 16:15
186Content type:Research
187Published on: 6 August 2024"
188"""
189# Chunk the text. The prob_threshold should be between (0, 1). The lower it is, the more chunks will be generated.
190# Therefore adjust it to your need, when prob_threshold is small like 0.000001, each token is one chunk,
191# when it is set to 1, the whole text will be one chunk, and, will be forced to choose a best possible position to chunk when it is about to exceed the max_tokens_per_chunk and no token satisfy the prob_threshold.
192chunks, token_pos = chunk_text_with_max_chunk_size(model, ad, tokenizer, prob_threshold=0.5, max_tokens_per_chunk = 400)
193
194# print chunks
195for i, (c, t) in enumerate(zip(chunks, token_pos)):
196 print(f"-----chunk: {i}----token_idx: {t}--------")
197 print(c)| Chunking | Size | Overlap | Recall | Precision | PrecisionΩ | IoU | Time complexity by token number N | Is max chunk size strictly controlable |
|---|---|---|---|---|---|---|---|---|
| Recursive | <= 800 | 400 | 85.4 ± 34.9 | 1.5 ± 1.3 | 6.7 ± 5.2 | 1.5 ± 1.3 | O(N) | Yes |
| TokenText | 800 | 400 | 87.9 ± 31.7 | 1.4 ± 1.1 | 4.7 ± 3.1 | 1.4 ± 1.1 | O(N) | Yes |
| Recursive | <= 400 | 200 | 88.1 ± 31.6 | 3.3 ± 2.7 | 13.9 ± 10.4 | 3.3 ± 2.7 | O(N) | Yes |
| TokenText | 400 | 200 | 88.6 ± 29.7 | 2.7 ± 2.2 | 8.4 ± 5.1 | 2.7 ± 2.2 | O(N) | Yes |
| Recursive | <= 400 | 0 | 89.5 ± 29.7 | 3.6 ± 3.2 | 17.7 ± 14.0 | 3.6 ± 3.2 | O(N) | Yes |
| TokenText | 400 | 0 | 89.2 ± 29.2 | 2.7 ± 2.2 | 12.5 ± 8.1 | 2.7 ± 2.2 | O(N) | Yes |
| Recursive | <= 200 | 0 | 88.1 ± 30.1 | 7.0 ± 5.6 | 29.9 ± 18.4 | 6.9 ± 5.6 | O(N) | Yes |
| TokenText | 200 | 0 | 87.0 ± 30.8 | 5.2 ± 4.1 | 21.0 ± 11.9 | 5.1 ± 4.1 | O(N) | Yes |
| Kamradt | N/A (~660) | 0 | 83.6 ± 36.8 | 1.5 ± 1.6 | 7.4 ± 10.2 | 1.5 ± 1.6 | O(N) | No |
| KamradtMod | <= 300 | 0 | 87.1 ± 31.9 | 2.1 ± 2.0 | 10.5 ± 12.3 | 2.1 ± 2.0 | O(N) | Yes |
| Cluster | 400 (~182) | 0 | 91.3 ± 25.4 | 4.5 ± 3.4 | 20.7 ± 14.5 | 4.5 ± 3.4 | O(N2) | No |
| Cluster | 200 (~103) | 0 | 87.3 ± 29.8 | 8.0 ± 6.0 | 34.0 ± 19.7 | 8.0 ± 6.0 | O(N2) | No |
| LLM (GPT4o) | N/A (~240) | 0 | 91.9 ± 26.5 | 3.9 ± 3.2 | 19.9 ± 16.3 | 3.9 ± 3.2 | O(N2) | No |
| semchunk | <=400 | 0 | 90.0 ± 29.1 | 3.6 ± 2.8 | 17.3 ± 12.6 | 3.6 ± 2.8 | O(N) | Yes |
| semchunk | <=200 | 0 | 89.3 ± 28.7 | 6.8 ± 5.2 | 28.9 ± 17.1 | 6.7 ± 5.1 | O(N) | Yes |
| ★ bert-chunker-3 (experimental, prob_threshold=0.50543) | <= 400 | 0 | 91.3 ± 26.6 | 5.4 ± 4.7 | 23.1 ± 17.6 | 5.4 ± 4.7 | O(N) | Yes |
| ★ bert-chunker-3 (experimental, prob_threshold=0.50543) | <= 200 | 0 | 89.7 ± 27.9 | 7.6 ± 6.0 | 30.9 ± 19.1 | 7.7 ± 5.8 | O(N) | Yes |
| ★ bert-chunker-3 (prob_threshold=0.50543) | N/A | 0 | 90.4 ± 28.7 | 3.3 ± 3.1 | 16.0 ± 17.0 | 3.3 ± 3.1 | O(N) | No |
1@article{bert-chunker,
2 title={bert-chunker: Efficient and Trained Chunking for Unstructured Documents},
3 author={Yannan Luo},
4 year={2024},
5 url={https://github.com/jackfsuia/bert-chunker}
6}