统计机器学习方法 for NLP：基于CRF的词性标注

tagged_sentence = nltk.corpus.treebank.tagged_sents(tagset='universal')
print("Number of Tagged Sentences ",len(tagged_sentence))
tagged_words=[tup for sent in tagged_sentence for tup in sent]
print("Total Number of Tagged words", len(tagged_words))
vocab=set([word for word,tag in tagged_words])
print("Vocabulary of the Corpus",len(vocab))
tags=set([tag for word,tag in tagged_words])
print("Number of Tags in the Corpus ",len(tags))
“”“
Number of Tagged Sentences  3914
Total Number of Tagged words 100676
Vocabulary of the Corpus 12408
Number of Tags in the Corpus  12
”“”
train_set, test_set = train_test_split(tagged_sentence,test_size=0.2,random_state=1234)
print("Number of Sentences in Training Data ",len(train_set))
print("Number of Sentences in Testing Data ",len(test_set))
“”“
Number of Sentences in Training Data  3131
Number of Sentences in Testing Data  783
”“”

# 特征定义
def features(sentence,index):
    ### sentence is of the form [w1,w2,w3,..], index is the position of the word in the sentence
    return {
        'is_first_capital':int(sentence[index][0].isupper()),
        'is_first_word': int(index==0),
        'is_last_word':int(index==len(sentence)-1),
        'is_complete_capital': int(sentence[index].upper()==sentence[index]),
        'prev_word':'' if index==0 else sentence[index-1],
        'next_word':'' if index==len(sentence)-1 else sentence[index+1],
        'is_numeric':int(sentence[index].isdigit()),
        'is_alphanumeric': int(bool((re.match('^(?=.*[0-9]$)(?=.*[a-zA-Z])',sentence[index])))),
        'prefix_1':sentence[index][0],
        'prefix_2': sentence[index][:2],
        'prefix_3':sentence[index][:3],
        'prefix_4':sentence[index][:4],
        'suffix_1':sentence[index][-1],
        'suffix_2':sentence[index][-2:],
        'suffix_3':sentence[index][-3:],
        'suffix_4':sentence[index][-4:],
        'word_has_hyphen': 1 if '-' in sentence[index] else 0
    }

def untag(sentence):
    return [word for word,tag in sentence]


def prepareData(tagged_sentences):
    X,y=[],[]
    for sentences in tagged_sentences:
        X.append([features(untag(sentences), index) for index in range(len(sentences))])
        y.append([tag for word,tag in sentences])
    return X,y

# 特征抽取
X_train,y_train=prepareData(train_set)
X_test,y_test=prepareData(test_set)

[{'is_alphanumeric': 0,
  'is_complete_capital': 0,
  'is_first_capital': 1,
  'is_first_word': 1,
  'is_last_word': 0,
  'is_numeric': 0,
  'next_word': 'Wall',
  'prefix_1': 'O',
  'prefix_2': 'On',
  'prefix_3': 'On',
  'prefix_4': 'On',
  'prev_word': '',
  'suffix_1': 'n',
  'suffix_2': 'On',
  'suffix_3': 'On',
  'suffix_4': 'On',
  'word_has_hyphen': 0},
 {'is_alphanumeric': 0,
  'is_complete_capital': 0,
  'is_first_capital': 1,
  'is_first_word': 0,
  'is_last_word': 0,
  'is_numeric': 0,
  'next_word': 'Street',
  'prefix_1': 'W',
  'prefix_2': 'Wa',
  'prefix_3': 'Wal',
  'prefix_4': 'Wall',
  'prev_word': 'On',
  'suffix_1': 'l',
  'suffix_2': 'll',
  'suffix_3': 'all',
  'suffix_4': 'Wall',
  'word_has_hyphen': 0},
...
]

['ADP',
 'NOUN',
 'NOUN',
...
]

from sklearn_crfsuite import CRF
crf = CRF(
    algorithm='lbfgs',
    c1=0.01,
    c2=0.1,
    max_iterations=100,
    all_possible_transitions=True
)
crf.fit(X_train, y_train)

y_pred=crf.predict(X_test)
metrics.flat_f1_score(y_test, y_pred,average='weighted',labels=crf.classes_)
“”“
0.9738471726864286
”“”

# 头部的转移特征
[(('ADJ', 'NOUN'), 4.114996),
 (('NOUN', 'NOUN'), 2.935448),
 (('NOUN', 'VERB'), 2.891987),
 (('VERB', 'PRT'), 2.519179),
 (('X', 'VERB'), 2.271558),
 (('ADP', 'NOUN'), 2.265833),
 (('NOUN', 'PRT'), 2.172849),
 (('PRON', 'VERB'), 2.117186),
 (('NUM', 'NOUN'), 2.059221),
 (('DET', 'NOUN'), 2.053832),
 (('ADV', 'VERB'), 1.994419),
 (('ADV', 'ADJ'), 1.957063),
 (('NOUN', 'ADP'), 1.838684),
 (('VERB', 'NOUN'), 1.763319),
 (('ADJ', 'ADJ'), 1.660578),
 (('NOUN', 'CONJ'), 1.591359),
 (('PRT', 'NOUN'), 1.398473),
 (('NOUN', '.'), 1.381863),
 (('NOUN', 'ADV'), 1.380086),
 (('ADV', 'ADV'), 1.301282)]
# 头部的状态特征
[(('prev_word:will', 'VERB'), 6.751359),
 (('prev_word:would', 'VERB'), 5.940819),
 (('prefix_1:*', 'X'), 5.830558),
 (('suffix_4:rest', 'NOUN'), 5.644523),
 (('suffix_2:ly', 'ADV'), 5.260228),
 (('is_first_capital', 'NOUN'), 5.043121),
 (('prev_word:could', 'VERB'), 5.018842),
 (('suffix_3:ous', 'ADJ'), 4.870949),
 (('prev_word:to', 'VERB'), 4.849822),
 (('suffix_4:will', 'VERB'), 4.677684),
 (('next_word:appeal', 'ADJ'), 4.386434),
 (('prev_word:how', 'PRT'), 4.35094),
 (('suffix_4:pany', 'NOUN'), 4.329975),
 (('prefix_4:many', 'ADJ'), 4.205028),
 (('prev_word:lock', 'PRT'), 4.153643),
 (('word_has_hyphen', 'ADJ'), 4.151036),
 (('prev_word:tune', 'PRT'), 4.147576),
 (('next_word:Express', 'NOUN'), 4.137127),
 (('suffix_4:food', 'NOUN'), 4.116688),
 (('suffix_2:ed', 'VERB'), 4.070659)]