[1] : ; . 2.1 2.4 [2 6, 9], 2.5 [7, 8].
. 2.1.
: () ().
[1]. . , .
[3] .
. (, , ), , , [1].
. ( ).
, .
(, , ).
. 2.1 (. 2.2).
.
, . [6].
, () .
, P(X = Xi) = pi, P(Y = Yj) = qj P(X = Xi; Y = Yj) = pij, , X Y,
I(X; Y) = Spij log2(pij/(piqj)).
i;j
I(X; X) = - Spi log2(pi).
i
H(X) = HX = I(X;X).
X.
.
1. I(X; Y) > 0, I(X; Y) = 0 , X Y .
ex-1 ³ x x 1 ³ lnx (x - 1)/ln2 ³ lg2x.
pij = pipj, ..
- I(X; Y) = Spij log2((piqj)/pij) = Spij (((piqj)/pij) -1)/ln2 = S(piqj - pij)/ln2 =
i;j i;j i;j
SpiSqj - Spij)/ln2 = (1 - 1)/ln2 = 0
i j i;j
2. I(X; Y) = I(Y; X) .
3. HX = 0, X . , X .
4. I(X; Y) = HX + HY - H(X; Y), H(X; Y) = Spij log2(pij).
i;j
,
Spij = pi, Spij = pj,
i j
HX = - Spi log2(pi) = - Spij log2(pi), HY = -Spj log2(pj) = - Spij log2(pj)
i i,j j i,j
HX + HY - H(X; Y) = Spij(- log2(pi) - log2(qj) + log2(pij)) = I(X; Y).
|
|
i,j
5. I(X; Y) ≤ I(X;X). I(X; Y) = I(X;X), TO X Y.
HY - H(X; Y) = Spij(-log2(qj) + log2(pij)) = Spij log2(pij/qj).
i,j i,j
pij = P(X = Xi; Y = Yj) ≤ qj = P(Y = Yj), pij/qj ≤ 1, 0, 0. HX = I(X;X) = I(X; Y), i pij 0 qj. pij = P(X = Xi; Y = Yj) = P(X = Xi| Y = Yj)P(Y = Yj) Î {q, 0} , P(X = Xi|Y = Yj) Î {0; 1}, , X Y.
.
2.1. 4 , ¼. .
HX = - Spi log2(pi) = - 4 ¼ log2 ¼ = 2
i
2.2. X 2.1 :
P(X = 1) = ¾; P(X = 2) = 1/8; P(X = 3) = P(X = 4) = 1/16.
1.
HX = ¾ log24/3 + 1/8 log28 +1/8 log216 = 19/8 ¾ log23 = 1.186 /.
2.3. X
X 1 2 3 4 5 6 7 8
p 0.1 0.2 0.1 0.05 0.1 0.05 0.3 0.1.
HX = 4 0.1 log210 + 0.2 log25 + 0,3 log210/3 + 2 0,05 log220 =
0:9 + log25 - 0:3 log23 = 2.75 /.
2.4. X , , Y , , , . I(X; Y) I I(Y; Y).
pi = P(X = i) = 1/6 i = 1, 6 qj = P(Y = j) j= 0, 1.
X 1 3 5 2 4 6 |1 3 5 2 4 6
Y 0 0 0 1 1 1 |1 1 1 0 0 0
p 1/6 | 0
I(X; Y) = Spij log2(pij/(piqj)) = 6 1/6 log22 = 1 /.
i;j
I(Y; Y) = Sqj log2(qj)) = 2 1/2 log22 = 1 /.
j
I(X; X) = Spi log2(pi)) = 6 1/6 log26 = 1 + log23 = 2.58 /.
i
I(X; Y) = I(Y; Y) , X Y, I(X; Y) ¹ I(X; X). Y X, X Y .
I(X; Y) .
H(X; Y) = - S pij log2 pij = log2 6 = 1 + log2 3 = HX,
i;j
I(X; Y) = HX + HY - HX = HY = 1 /.
, , .
() . . .
in f(s) = - log2 p(s)
s ( ), , p(s) s.
.
1. (s1 s2) , inf(s1) ³ inf(s2).
2. inf(s) ³ 0.
3. s - , inf(s) = 0.
4. inf(s1s2) = inf(s1) + inf(s2) p(s1 s2) = p(s1)p(s2), .. s1 s2 .
s1 > a > 3 s2 = a = 7 , inf(s2) > inf(s1) s2 , s1.
cont(s) = 1 - p(s). cont(s) = 1 2-inf(s) in f(s) = - log2(1 - cont(s)).