Spaces:
Running
Running
add box which shows encoded tokens, also add labels
#1
by
xzuyn - opened
app.py
CHANGED
|
@@ -5,7 +5,8 @@ sp = SentencePieceProcessor(model_file="tokenizer.model")
|
|
| 5 |
|
| 6 |
def tokenize(input_text):
|
| 7 |
tokens = sp.EncodeAsIds(input_text)
|
| 8 |
-
|
|
|
|
| 9 |
|
| 10 |
-
iface = gr.Interface(fn=tokenize, inputs=gr.inputs.Textbox(lines=7), outputs="
|
| 11 |
-
iface.launch()
|
|
|
|
| 5 |
|
| 6 |
def tokenize(input_text):
|
| 7 |
tokens = sp.EncodeAsIds(input_text)
|
| 8 |
+
decoded_tokens = sp.DecodeIds(tokens)
|
| 9 |
+
return len(tokens), tokens
|
| 10 |
|
| 11 |
+
iface = gr.Interface(fn=tokenize, inputs=gr.inputs.Textbox(lines=7, label="Input Text"), outputs=[gr.outputs.Textbox(label="Token Count"), gr.outputs.Textbox(label="Tokens")])
|
| 12 |
+
iface.launch()
|