Upload tokenizer

Browse files

Files changed (6) hide show

added_tokens.json +218 -0
merges.txt +0 -0
special_tokens_map.json +330 -0
tokenizer.json +0 -0
tokenizer_config.json +65 -0
vocab.json +0 -0

added_tokens.json ADDED Viewed

	@@ -0,0 +1,218 @@

+{
+  " <C>": 32100,
+  " <M>": 32101,
+  " <decl>": 32102,
+  " <gF>": 32103,
+  " <gL>": 32104,
+  " <multi>": 32105,
+  " <pF>": 32106,
+  " <pL>": 32107,
+  " <ret>": 32108,
+  " <throw>": 32109,
+  " AALOAD": 32110,
+  " AASTORE": 32111,
+  " ACONST_NULL": 32112,
+  " ALOAD": 32113,
+  " ALOAD_0": 32114,
+  " ALOAD_1": 32115,
+  " ALOAD_2": 32116,
+  " ALOAD_3": 32117,
+  " ANEWARRAY": 32118,
+  " ARETURN": 32119,
+  " ARRAYLENGTH": 32120,
+  " ASTORE": 32121,
+  " ASTORE_0": 32122,
+  " ASTORE_1": 32123,
+  " ASTORE_2": 32124,
+  " ASTORE_3": 32125,
+  " ATHROW": 32126,
+  " BALOAD": 32127,
+  " BASTORE": 32128,
+  " BIPUSH": 32129,
+  " CALOAD": 32130,
+  " CASTORE": 32131,
+  " CHECKCAST": 32132,
+  " D2F": 32133,
+  " D2I": 32134,
+  " D2L": 32135,
+  " DADD": 32136,
+  " DALOAD": 32137,
+  " DASTORE": 32138,
+  " DCMPG": 32139,
+  " DCMPL": 32140,
+  " DCONST_0": 32141,
+  " DCONST_1": 32142,
+  " DDIV": 32143,
+  " DLOAD": 32144,
+  " DLOAD_0": 32145,
+  " DLOAD_1": 32146,
+  " DLOAD_2": 32147,
+  " DLOAD_3": 32148,
+  " DMUL": 32149,
+  " DNEG": 32150,
+  " DREM": 32151,
+  " DRETURN": 32152,
+  " DSTORE": 32153,
+  " DSTORE_0": 32154,
+  " DSTORE_1": 32155,
+  " DSTORE_2": 32156,
+  " DSTORE_3": 32157,
+  " DSUB": 32158,
+  " DUP": 32159,
+  " DUP2": 32160,
+  " DUP2_X1": 32161,
+  " DUP2_X2": 32162,
+  " DUP_X1": 32163,
+  " DUP_X2": 32164,
+  " F2D": 32165,
+  " F2I": 32166,
+  " F2L": 32167,
+  " FADD": 32168,
+  " FALOAD": 32169,
+  " FASTORE": 32170,
+  " FCMPG": 32171,
+  " FCMPL": 32172,
+  " FCONST_0": 32173,
+  " FCONST_1": 32174,
+  " FCONST_2": 32175,
+  " FDIV": 32176,
+  " FLOAD": 32177,
+  " FLOAD_0": 32178,
+  " FLOAD_1": 32179,
+  " FLOAD_2": 32180,
+  " FLOAD_3": 32181,
+  " FMUL": 32182,
+  " FNEG": 32183,
+  " FREM": 32184,
+  " FRETURN": 32185,
+  " FSTORE": 32186,
+  " FSTORE_0": 32187,
+  " FSTORE_1": 32188,
+  " FSTORE_2": 32189,
+  " FSTORE_3": 32190,
+  " FSUB": 32191,
+  " GETFIELD": 32192,
+  " GETSTATIC": 32193,
+  " GOTO": 32194,
+  " GOTO_W": 32195,
+  " I2B": 32196,
+  " I2C": 32197,
+  " I2D": 32198,
+  " I2F": 32199,
+  " I2L": 32200,
+  " I2S": 32201,
+  " IADD": 32202,
+  " IALOAD": 32203,
+  " IAND": 32204,
+  " IASTORE": 32205,
+  " ICONST_0": 32206,
+  " ICONST_1": 32207,
+  " ICONST_2": 32208,
+  " ICONST_3": 32209,
+  " ICONST_4": 32210,
+  " ICONST_5": 32211,
+  " ICONST_M1": 32212,
+  " IDIV": 32213,
+  " IFEQ": 32214,
+  " IFGE": 32215,
+  " IFGT": 32216,
+  " IFLE": 32217,
+  " IFLT": 32218,
+  " IFNE": 32219,
+  " IFNONNULL": 32220,
+  " IFNULL": 32221,
+  " IF_ACMPEQ": 32222,
+  " IF_ACMPNE": 32223,
+  " IF_ICMPEQ": 32224,
+  " IF_ICMPGE": 32225,
+  " IF_ICMPGT": 32226,
+  " IF_ICMPLE": 32227,
+  " IF_ICMPLT": 32228,
+  " IF_ICMPNE": 32229,
+  " IINC": 32230,
+  " ILOAD": 32231,
+  " ILOAD_0": 32232,
+  " ILOAD_1": 32233,
+  " ILOAD_2": 32234,
+  " ILOAD_3": 32235,
+  " IMUL": 32236,
+  " INEG": 32237,
+  " INSTANCEOF": 32238,
+  " INVOKEDYNAMIC": 32239,
+  " INVOKEINTERFACE": 32240,
+  " INVOKESPECIAL": 32241,
+  " INVOKESTATIC": 32242,
+  " INVOKEVIRTUAL": 32243,
+  " IOR": 32244,
+  " IREM": 32245,
+  " IRETURN": 32246,
+  " ISHL": 32247,
+  " ISHR": 32248,
+  " ISTORE": 32249,
+  " ISTORE_0": 32250,
+  " ISTORE_1": 32251,
+  " ISTORE_2": 32252,
+  " ISTORE_3": 32253,
+  " ISUB": 32254,
+  " IUSHR": 32255,
+  " IXOR": 32256,
+  " JSR": 32257,
+  " JSR_W": 32258,
+  " L2D": 32259,
+  " L2F": 32260,
+  " L2I": 32261,
+  " LADD": 32262,
+  " LALOAD": 32263,
+  " LAND": 32264,
+  " LASTORE": 32265,
+  " LCMP": 32266,
+  " LCONST_0": 32267,
+  " LCONST_1": 32268,
+  " LDC": 32269,
+  " LDC2_W": 32270,
+  " LDC_END": 32271,
+  " LDC_W": 32272,
+  " LDIV": 32273,
+  " LLOAD": 32274,
+  " LLOAD_0": 32275,
+  " LLOAD_1": 32276,
+  " LLOAD_2": 32277,
+  " LLOAD_3": 32278,
+  " LMUL": 32279,
+  " LNEG": 32280,
+  " LOOKUPSWITCH": 32281,
+  " LOR": 32282,
+  " LREM": 32283,
+  " LRETURN": 32284,
+  " LSHL": 32285,
+  " LSHR": 32286,
+  " LSTORE": 32287,
+  " LSTORE_0": 32288,
+  " LSTORE_1": 32289,
+  " LSTORE_2": 32290,
+  " LSTORE_3": 32291,
+  " LSUB": 32292,
+  " LUSHR": 32293,
+  " LXOR": 32294,
+  " MONITORENTER": 32295,
+  " MONITOREXIT": 32296,
+  " MULTIANEWARRAY": 32297,
+  " NEW": 32298,
+  " NEWARRAY": 32299,
+  " NOP": 32300,
+  " POP": 32301,
+  " POP2": 32302,
+  " PUTFIELD": 32303,
+  " PUTSTATIC": 32304,
+  " RET": 32305,
+  " RETURN": 32306,
+  " SALOAD": 32307,
+  " SASTORE": 32308,
+  " SIPUSH": 32309,
+  " SWAP": 32310,
+  " TABLESWITCH": 32311,
+  " WIDE": 32312,
+  "<cls>": 32314,
+  "<sep>": 32315,
+  "<|UNKNOWN|>": 32313
+}

merges.txt ADDED Viewed

The diff for this file is too large to render. See raw diff

special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,330 @@

+{
+  "additional_special_tokens": [
+    " <C>",
+    " <M>",
+    " <decl>",
+    " <gF>",
+    " <gL>",
+    " <multi>",
+    " <pF>",
+    " <pL>",
+    " <ret>",
+    " <throw>",
+    " AALOAD",
+    " AASTORE",
+    " ACONST_NULL",
+    " ALOAD",
+    " ALOAD_0",
+    " ALOAD_1",
+    " ALOAD_2",
+    " ALOAD_3",
+    " ANEWARRAY",
+    " ARETURN",
+    " ARRAYLENGTH",
+    " ASTORE",
+    " ASTORE_0",
+    " ASTORE_1",
+    " ASTORE_2",
+    " ASTORE_3",
+    " ATHROW",
+    " BALOAD",
+    " BASTORE",
+    " BIPUSH",
+    " CALOAD",
+    " CASTORE",
+    " CHECKCAST",
+    " D2F",
+    " D2I",
+    " D2L",
+    " DADD",
+    " DALOAD",
+    " DASTORE",
+    " DCMPG",
+    " DCMPL",
+    " DCONST_0",
+    " DCONST_1",
+    " DDIV",
+    " DLOAD",
+    " DLOAD_0",
+    " DLOAD_1",
+    " DLOAD_2",
+    " DLOAD_3",
+    " DMUL",
+    " DNEG",
+    " DREM",
+    " DRETURN",
+    " DSTORE",
+    " DSTORE_0",
+    " DSTORE_1",
+    " DSTORE_2",
+    " DSTORE_3",
+    " DSUB",
+    " DUP",
+    " DUP2",
+    " DUP2_X1",
+    " DUP2_X2",
+    " DUP_X1",
+    " DUP_X2",
+    " F2D",
+    " F2I",
+    " F2L",
+    " FADD",
+    " FALOAD",
+    " FASTORE",
+    " FCMPG",
+    " FCMPL",
+    " FCONST_0",
+    " FCONST_1",
+    " FCONST_2",
+    " FDIV",
+    " FLOAD",
+    " FLOAD_0",
+    " FLOAD_1",
+    " FLOAD_2",
+    " FLOAD_3",
+    " FMUL",
+    " FNEG",
+    " FREM",
+    " FRETURN",
+    " FSTORE",
+    " FSTORE_0",
+    " FSTORE_1",
+    " FSTORE_2",
+    " FSTORE_3",
+    " FSUB",
+    " GETFIELD",
+    " GETSTATIC",
+    " GOTO",
+    " GOTO_W",
+    " I2B",
+    " I2C",
+    " I2D",
+    " I2F",
+    " I2L",
+    " I2S",
+    " IADD",
+    " IALOAD",
+    " IAND",
+    " IASTORE",
+    " ICONST_0",
+    " ICONST_1",
+    " ICONST_2",
+    " ICONST_3",
+    " ICONST_4",
+    " ICONST_5",
+    " ICONST_M1",
+    " IDIV",
+    " IFEQ",
+    " IFGE",
+    " IFGT",
+    " IFLE",
+    " IFLT",
+    " IFNE",
+    " IFNONNULL",
+    " IFNULL",
+    " IF_ACMPEQ",
+    " IF_ACMPNE",
+    " IF_ICMPEQ",
+    " IF_ICMPGE",
+    " IF_ICMPGT",
+    " IF_ICMPLE",
+    " IF_ICMPLT",
+    " IF_ICMPNE",
+    " IINC",
+    " ILOAD",
+    " ILOAD_0",
+    " ILOAD_1",
+    " ILOAD_2",
+    " ILOAD_3",
+    " IMUL",
+    " INEG",
+    " INSTANCEOF",
+    " INVOKEDYNAMIC",
+    " INVOKEINTERFACE",
+    " INVOKESPECIAL",
+    " INVOKESTATIC",
+    " INVOKEVIRTUAL",
+    " IOR",
+    " IREM",
+    " IRETURN",
+    " ISHL",
+    " ISHR",
+    " ISTORE",
+    " ISTORE_0",
+    " ISTORE_1",
+    " ISTORE_2",
+    " ISTORE_3",
+    " ISUB",
+    " IUSHR",
+    " IXOR",
+    " JSR",
+    " JSR_W",
+    " L2D",
+    " L2F",
+    " L2I",
+    " LADD",
+    " LALOAD",
+    " LAND",
+    " LASTORE",
+    " LCMP",
+    " LCONST_0",
+    " LCONST_1",
+    " LDC",
+    " LDC2_W",
+    " LDC_END",
+    " LDC_W",
+    " LDIV",
+    " LLOAD",
+    " LLOAD_0",
+    " LLOAD_1",
+    " LLOAD_2",
+    " LLOAD_3",
+    " LMUL",
+    " LNEG",
+    " LOOKUPSWITCH",
+    " LOR",
+    " LREM",
+    " LRETURN",
+    " LSHL",
+    " LSHR",
+    " LSTORE",
+    " LSTORE_0",
+    " LSTORE_1",
+    " LSTORE_2",
+    " LSTORE_3",
+    " LSUB",
+    " LUSHR",
+    " LXOR",
+    " MONITORENTER",
+    " MONITOREXIT",
+    " MULTIANEWARRAY",
+    " NEW",
+    " NEWARRAY",
+    " NOP",
+    " POP",
+    " POP2",
+    " PUTFIELD",
+    " PUTSTATIC",
+    " RET",
+    " RETURN",
+    " SALOAD",
+    " SASTORE",
+    " SIPUSH",
+    " SWAP",
+    " TABLESWITCH",
+    " WIDE",
+    "<extra_id_0>",
+    "<extra_id_10>",
+    "<extra_id_11>",
+    "<extra_id_12>",
+    "<extra_id_13>",
+    "<extra_id_14>",
+    "<extra_id_15>",
+    "<extra_id_16>",
+    "<extra_id_17>",
+    "<extra_id_18>",
+    "<extra_id_19>",
+    "<extra_id_1>",
+    "<extra_id_20>",
+    "<extra_id_21>",
+    "<extra_id_22>",
+    "<extra_id_23>",
+    "<extra_id_24>",
+    "<extra_id_25>",
+    "<extra_id_26>",
+    "<extra_id_27>",
+    "<extra_id_28>",
+    "<extra_id_29>",
+    "<extra_id_2>",
+    "<extra_id_30>",
+    "<extra_id_31>",
+    "<extra_id_32>",
+    "<extra_id_33>",
+    "<extra_id_34>",
+    "<extra_id_35>",
+    "<extra_id_36>",
+    "<extra_id_37>",
+    "<extra_id_38>",
+    "<extra_id_39>",
+    "<extra_id_3>",
+    "<extra_id_40>",
+    "<extra_id_41>",
+    "<extra_id_42>",
+    "<extra_id_43>",
+    "<extra_id_44>",
+    "<extra_id_45>",
+    "<extra_id_46>",
+    "<extra_id_47>",
+    "<extra_id_48>",
+    "<extra_id_49>",
+    "<extra_id_4>",
+    "<extra_id_50>",
+    "<extra_id_51>",
+    "<extra_id_52>",
+    "<extra_id_53>",
+    "<extra_id_54>",
+    "<extra_id_55>",
+    "<extra_id_56>",
+    "<extra_id_57>",
+    "<extra_id_58>",
+    "<extra_id_59>",
+    "<extra_id_5>",
+    "<extra_id_60>",
+    "<extra_id_61>",
+    "<extra_id_62>",
+    "<extra_id_63>",
+    "<extra_id_64>",
+    "<extra_id_65>",
+    "<extra_id_66>",
+    "<extra_id_67>",
+    "<extra_id_68>",
+    "<extra_id_69>",
+    "<extra_id_6>",
+    "<extra_id_70>",
+    "<extra_id_71>",
+    "<extra_id_72>",
+    "<extra_id_73>",
+    "<extra_id_74>",
+    "<extra_id_75>",
+    "<extra_id_76>",
+    "<extra_id_77>",
+    "<extra_id_78>",
+    "<extra_id_79>",
+    "<extra_id_7>",
+    "<extra_id_80>",
+    "<extra_id_81>",
+    "<extra_id_82>",
+    "<extra_id_83>",
+    "<extra_id_84>",
+    "<extra_id_85>",
+    "<extra_id_86>",
+    "<extra_id_87>",
+    "<extra_id_88>",
+    "<extra_id_89>",
+    "<extra_id_8>",
+    "<extra_id_90>",
+    "<extra_id_91>",
+    "<extra_id_92>",
+    "<extra_id_93>",
+    "<extra_id_94>",
+    "<extra_id_95>",
+    "<extra_id_96>",
+    "<extra_id_97>",
+    "<extra_id_98>",
+    "<extra_id_99>",
+    "<extra_id_9>"
+  ],
+  "bos_token": "<s>",
+  "cls_token": "<cls>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "<sep>",
+  "unk_token": "<|UNKNOWN|>"
+}

tokenizer.json ADDED Viewed

The diff for this file is too large to render. See raw diff

tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,65 @@

+{
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "mask_token": {
+    "__type": "AddedToken",
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "model_max_length": 512,
+  "name_or_path": "_work/subtokenizer/codet5",
+  "pad_token": {
+    "__type": "AddedToken",
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "special_tokens_map_file": "/home/pynie/.cache/huggingface/transformers/5941df5e4315c5ab63b7b2ac791fb0bf0f209744a055c06b43b5274849137cdd.b9905d0575bde443a20834122b6e2d48e853b2e36444ce98ddeb43c38097eb3f",
+  "tokenizer_class": "RobertaTokenizer",
+  "trim_offsets": true,
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

vocab.json ADDED Viewed

The diff for this file is too large to render. See raw diff