diff --git a/v1.13.x/en/guide/custom-routes/index.html b/v1.13.x/en/guide/custom-routes/index.html
index cd533f891..9424cde0f 100644
--- a/v1.13.x/en/guide/custom-routes/index.html
+++ b/v1.13.x/en/guide/custom-routes/index.html
@@ -17,7 +17,7 @@
 <span class=w>        </span><span class=no>&quot;gw&quot;: &quot;10.16.0.254&quot;</span>
 <span class=w>      </span><span class=no>}, {</span>
 <span class=w>        </span><span class=no>&quot;gw&quot;: &quot;10.16.0.254&quot;</span>
-<span class=w>      </span><span class=no>}]&#39;</span>
+<span class=w>      </span><span class=no>}]</span>
 <span class=nt>spec</span><span class=p>:</span>
 <span class=w>  </span><span class=nt>containers</span><span class=p>:</span>
 <span class=w>  </span><span class="p p-Indicator">-</span><span class=w> </span><span class=nt>name</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">nginx</span>
@@ -44,12 +44,12 @@
 <span class=w>            </span><span class=no>&quot;gw&quot;: &quot;10.16.0.254&quot;</span>
 <span class=w>          </span><span class=no>}, {</span>
 <span class=w>            </span><span class=no>&quot;gw&quot;: &quot;10.16.0.254&quot;</span>
-<span class=w>          </span><span class=no>}]&#39;</span>
+<span class=w>          </span><span class=no>}]</span>
 <span class=w>    </span><span class=nt>spec</span><span class=p>:</span>
 <span class=w>      </span><span class=nt>containers</span><span class=p>:</span>
 <span class=w>      </span><span class="p p-Indicator">-</span><span class=w> </span><span class=nt>name</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">nginx</span>
 <span class=w>        </span><span class=nt>image</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">docker.io/library/nginx:alpine</span>
-</code></pre></div> <p><a class=md-button href=https://ma.alauda.cn/p/0f39a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M9.5 4C5.36 4 2 6.69 2 10c0 1.89 1.08 3.56 2.78 4.66L4 17l2.5-1.5c.89.31 1.87.5 2.91.5A5.22 5.22 0 0 1 9 14c0-3.31 3.13-6 7-6 .19 0 .38 0 .56.03C15.54 5.69 12.78 4 9.5 4m-3 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m5 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1M16 9c-3.31 0-6 2.24-6 5s2.69 5 6 5c.67 0 1.31-.08 1.91-.25L20 20l-.62-1.87C20.95 17.22 22 15.71 22 14c0-2.76-2.69-5-6-5m-2 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m4 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1Z"/></svg></span> 微信群</a> <a class=md-button href=https://communityinviter.com/apps/kube-ovn/kube-ovn/ target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M6 15a2 2 0 0 1-2 2 2 2 0 0 1-2-2 2 2 0 0 1 2-2h2v2m1 0a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-5m2-8a2 2 0 0 1-2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2v2H9m0 1a2 2 0 0 1 2 2 2 2 0 0 1-2 2H4a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5m8 2a2 2 0 0 1 2-2 2 2 0 0 1 2 2 2 2 0 0 1-2 2h-2v-2m-1 0a2 2 0 0 1-2 2 2 2 0 0 1-2-2V5a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5m-2 8a2 2 0 0 1 2 2 2 2 0 0 1-2 2 2 2 0 0 1-2-2v-2h2m0-1a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-5Z"/></svg></span> Slack</a> <a class=md-button href=https://twitter.com/KubeOvn target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M22.46 6c-.77.35-1.6.58-2.46.69.88-.53 1.56-1.37 1.88-2.38-.83.5-1.75.85-2.72 1.05C18.37 4.5 17.26 4 16 4c-2.35 0-4.27 1.92-4.27 4.29 0 .34.04.67.11.98C8.28 9.09 5.11 7.38 3 4.79c-.37.63-.58 1.37-.58 2.15 0 1.49.75 2.81 1.91 3.56-.71 0-1.37-.2-1.95-.5v.03c0 2.08 1.48 3.82 3.44 4.21a4.22 4.22 0 0 1-1.93.07 4.28 4.28 0 0 0 4 2.98 8.521 8.521 0 0 1-5.33 1.84c-.34 0-.68-.02-1.02-.06C3.44 20.29 5.7 21 8.12 21 16 21 20.33 14.46 20.33 8.79c0-.19 0-.37-.01-.56.84-.6 1.56-1.36 2.14-2.23Z"/></svg></span> Twitter</a> <a class=md-button href=https://ma.alauda.cn/p/2f53a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="m20 8-8 5-8-5V6l8 5 8-5m0-2H4c-1.11 0-2 .89-2 2v12a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V6a2 2 0 0 0-2-2Z"/></svg></span> Support</a></p> <hr> <div class=md-source-file> <small> Last update: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">May 9, 2023</span> <br> Created: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">February 16, 2023</span> </small> </div> <h2 id=__comments>Comments</h2> <script src=https://giscus.app/client.js data-repo=kubeovn/kube-ovn data-repo-id="MDEwOlJlcG9zaXRvcnkxNzcwNjg5NjE=" data-category=Announcements data-category-id=DIC_kwDOCo3boc4CAj60 data-mapping=pathname data-reactions-enabled=1 data-emit-metadata=0 data-input-position=bottom data-theme=light data-lang=en crossorigin=anonymous async>
+</code></pre></div> <p><a class=md-button href=https://ma.alauda.cn/p/0f39a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M9.5 4C5.36 4 2 6.69 2 10c0 1.89 1.08 3.56 2.78 4.66L4 17l2.5-1.5c.89.31 1.87.5 2.91.5A5.22 5.22 0 0 1 9 14c0-3.31 3.13-6 7-6 .19 0 .38 0 .56.03C15.54 5.69 12.78 4 9.5 4m-3 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m5 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1M16 9c-3.31 0-6 2.24-6 5s2.69 5 6 5c.67 0 1.31-.08 1.91-.25L20 20l-.62-1.87C20.95 17.22 22 15.71 22 14c0-2.76-2.69-5-6-5m-2 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m4 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1Z"/></svg></span> 微信群</a> <a class=md-button href=https://communityinviter.com/apps/kube-ovn/kube-ovn/ target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M6 15a2 2 0 0 1-2 2 2 2 0 0 1-2-2 2 2 0 0 1 2-2h2v2m1 0a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-5m2-8a2 2 0 0 1-2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2v2H9m0 1a2 2 0 0 1 2 2 2 2 0 0 1-2 2H4a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5m8 2a2 2 0 0 1 2-2 2 2 0 0 1 2 2 2 2 0 0 1-2 2h-2v-2m-1 0a2 2 0 0 1-2 2 2 2 0 0 1-2-2V5a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5m-2 8a2 2 0 0 1 2 2 2 2 0 0 1-2 2 2 2 0 0 1-2-2v-2h2m0-1a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-5Z"/></svg></span> Slack</a> <a class=md-button href=https://twitter.com/KubeOvn target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M22.46 6c-.77.35-1.6.58-2.46.69.88-.53 1.56-1.37 1.88-2.38-.83.5-1.75.85-2.72 1.05C18.37 4.5 17.26 4 16 4c-2.35 0-4.27 1.92-4.27 4.29 0 .34.04.67.11.98C8.28 9.09 5.11 7.38 3 4.79c-.37.63-.58 1.37-.58 2.15 0 1.49.75 2.81 1.91 3.56-.71 0-1.37-.2-1.95-.5v.03c0 2.08 1.48 3.82 3.44 4.21a4.22 4.22 0 0 1-1.93.07 4.28 4.28 0 0 0 4 2.98 8.521 8.521 0 0 1-5.33 1.84c-.34 0-.68-.02-1.02-.06C3.44 20.29 5.7 21 8.12 21 16 21 20.33 14.46 20.33 8.79c0-.19 0-.37-.01-.56.84-.6 1.56-1.36 2.14-2.23Z"/></svg></span> Twitter</a> <a class=md-button href=https://ma.alauda.cn/p/2f53a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="m20 8-8 5-8-5V6l8 5 8-5m0-2H4c-1.11 0-2 .89-2 2v12a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V6a2 2 0 0 0-2-2Z"/></svg></span> Support</a></p> <hr> <div class=md-source-file> <small> Last update: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">September 26, 2023</span> <br> Created: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">February 16, 2023</span> </small> </div> <h2 id=__comments>Comments</h2> <script src=https://giscus.app/client.js data-repo=kubeovn/kube-ovn data-repo-id="MDEwOlJlcG9zaXRvcnkxNzcwNjg5NjE=" data-category=Announcements data-category-id=DIC_kwDOCo3boc4CAj60 data-mapping=pathname data-reactions-enabled=1 data-emit-metadata=0 data-input-position=bottom data-theme=light data-lang=en crossorigin=anonymous async>
 </script> <script>
     var palette = __md_get("__palette")
     if (palette && typeof palette.color === "object")
diff --git a/v1.13.x/en/guide/static-ip-mac/index.html b/v1.13.x/en/guide/static-ip-mac/index.html
index f3061346b..b9e2c8a27 100644
--- a/v1.13.x/en/guide/static-ip-mac/index.html
+++ b/v1.13.x/en/guide/static-ip-mac/index.html
@@ -27,7 +27,7 @@
 <span class=w>      </span><span class=nt>containers</span><span class=p>:</span>
 <span class=w>        </span><span class="p p-Indicator">-</span><span class=w> </span><span class=nt>name</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">ippool</span>
 <span class=w>          </span><span class=nt>image</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">docker.io/library/nginx:alpine</span>
-</code></pre></div> <p>The following points need to be noted when using annotation.</p> <ol> <li>The IP/Mac used cannot conflict with an existing IP/Mac.</li> <li>The IP must be in the CIDR range of the Subnet it belongs to.</li> <li>You can specify only IP or Mac. When you specify only one, the other one will be assigned randomly.</li> </ol> <h2 id=workload-ip-pool>Workload IP Pool<a class=headerlink href=#workload-ip-pool title="Permanent link">&para;</a></h2> <p>Kube-OVN supports setting fixed IPs for Workloads (Deployment/StatefulSet/DaemonSet/Job/CronJob) via annotation <code>ovn.kubernetes.io/ip_pool</code>. <code>kube-ovn-controllerr</code> will automatically select the IP specified in <code>ovn.kubernetes.io/ip_pool</code> and perform conflict detection.</p> <p>The Annotation of the IP Pool needs to be added to the <code>annotation</code> field in the <code>template</code>. In addition to Kubernetes built-in workload types, other user-defined workloads can also be assigned fixed addresses using the same approach.</p> <h3 id=deployment-with-fixed-ips>Deployment With Fixed IPs<a class=headerlink href=#deployment-with-fixed-ips title="Permanent link">&para;</a></h3> <div class=highlight><pre><span></span><code><span class=nt>apiVersion</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">apps/v1</span>
+</code></pre></div> <p>The following points need to be noted when using annotation.</p> <ol> <li>The IP/Mac used cannot conflict with an existing IP/Mac.</li> <li>The IP must be in the CIDR range of the Subnet it belongs to.</li> <li>You can specify only IP or Mac. When you specify only one, the other one will be assigned randomly.</li> </ol> <h2 id=workload-ip-pool>Workload IP Pool<a class=headerlink href=#workload-ip-pool title="Permanent link">&para;</a></h2> <p>Kube-OVN supports setting fixed IPs for Workloads (Deployment/StatefulSet/DaemonSet/Job/CronJob) via annotation <code>ovn.kubernetes.io/ip_pool</code>. <code>kube-ovn-controller</code> will automatically select the IP specified in <code>ovn.kubernetes.io/ip_pool</code> and perform conflict detection.</p> <p>The Annotation of the IP Pool needs to be added to the <code>annotation</code> field in the <code>template</code>. In addition to Kubernetes built-in workload types, other user-defined workloads can also be assigned fixed addresses using the same approach.</p> <h3 id=deployment-with-fixed-ips>Deployment With Fixed IPs<a class=headerlink href=#deployment-with-fixed-ips title="Permanent link">&para;</a></h3> <div class=highlight><pre><span></span><code><span class=nt>apiVersion</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">apps/v1</span>
 <span class=nt>kind</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">Deployment</span>
 <span class=nt>metadata</span><span class=p>:</span>
 <span class=w>  </span><span class=nt>namespace</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">ls1</span>
@@ -70,7 +70,7 @@
 <span class=w>        </span><span class=nt>ports</span><span class=p>:</span>
 <span class=w>        </span><span class="p p-Indicator">-</span><span class=w> </span><span class=nt>containerPort</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">80</span>
 <span class=w>          </span><span class=nt>name</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">web</span>
-</code></pre></div> <p>You can try to delete the Pod under StatefulSet to observe if the Pod IP changes.</p> <h2 id=kubevirt-vm-fixed-address>KubeVirt VM Fixed Address<a class=headerlink href=#kubevirt-vm-fixed-address title="Permanent link">&para;</a></h2> <p>For VM instances created by KubeVirt, <code>kube-ovn-controller</code> can assign and manage IP addresses in a similar way to the StatefulSet Pod. This allows VM instances address fixed during start-up, shutdown, upgrade, migration, and other operations throughout their lifecycle, making them more compatible with the actual virtualization user experience.</p> <p><a class=md-button href=https://ma.alauda.cn/p/0f39a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M9.5 4C5.36 4 2 6.69 2 10c0 1.89 1.08 3.56 2.78 4.66L4 17l2.5-1.5c.89.31 1.87.5 2.91.5A5.22 5.22 0 0 1 9 14c0-3.31 3.13-6 7-6 .19 0 .38 0 .56.03C15.54 5.69 12.78 4 9.5 4m-3 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m5 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1M16 9c-3.31 0-6 2.24-6 5s2.69 5 6 5c.67 0 1.31-.08 1.91-.25L20 20l-.62-1.87C20.95 17.22 22 15.71 22 14c0-2.76-2.69-5-6-5m-2 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m4 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1Z"/></svg></span> 微信群</a> <a class=md-button href=https://communityinviter.com/apps/kube-ovn/kube-ovn/ target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M6 15a2 2 0 0 1-2 2 2 2 0 0 1-2-2 2 2 0 0 1 2-2h2v2m1 0a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-5m2-8a2 2 0 0 1-2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2v2H9m0 1a2 2 0 0 1 2 2 2 2 0 0 1-2 2H4a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5m8 2a2 2 0 0 1 2-2 2 2 0 0 1 2 2 2 2 0 0 1-2 2h-2v-2m-1 0a2 2 0 0 1-2 2 2 2 0 0 1-2-2V5a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5m-2 8a2 2 0 0 1 2 2 2 2 0 0 1-2 2 2 2 0 0 1-2-2v-2h2m0-1a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-5Z"/></svg></span> Slack</a> <a class=md-button href=https://twitter.com/KubeOvn target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M22.46 6c-.77.35-1.6.58-2.46.69.88-.53 1.56-1.37 1.88-2.38-.83.5-1.75.85-2.72 1.05C18.37 4.5 17.26 4 16 4c-2.35 0-4.27 1.92-4.27 4.29 0 .34.04.67.11.98C8.28 9.09 5.11 7.38 3 4.79c-.37.63-.58 1.37-.58 2.15 0 1.49.75 2.81 1.91 3.56-.71 0-1.37-.2-1.95-.5v.03c0 2.08 1.48 3.82 3.44 4.21a4.22 4.22 0 0 1-1.93.07 4.28 4.28 0 0 0 4 2.98 8.521 8.521 0 0 1-5.33 1.84c-.34 0-.68-.02-1.02-.06C3.44 20.29 5.7 21 8.12 21 16 21 20.33 14.46 20.33 8.79c0-.19 0-.37-.01-.56.84-.6 1.56-1.36 2.14-2.23Z"/></svg></span> Twitter</a> <a class=md-button href=https://ma.alauda.cn/p/2f53a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="m20 8-8 5-8-5V6l8 5 8-5m0-2H4c-1.11 0-2 .89-2 2v12a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V6a2 2 0 0 0-2-2Z"/></svg></span> Support</a></p> <hr> <div class=md-source-file> <small> Last update: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">September 7, 2023</span> <br> Created: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">May 20, 2022</span> </small> </div> <h2 id=__comments>Comments</h2> <script src=https://giscus.app/client.js data-repo=kubeovn/kube-ovn data-repo-id="MDEwOlJlcG9zaXRvcnkxNzcwNjg5NjE=" data-category=Announcements data-category-id=DIC_kwDOCo3boc4CAj60 data-mapping=pathname data-reactions-enabled=1 data-emit-metadata=0 data-input-position=bottom data-theme=light data-lang=en crossorigin=anonymous async>
+</code></pre></div> <p>You can try to delete the Pod under StatefulSet to observe if the Pod IP changes.</p> <h2 id=kubevirt-vm-fixed-address>KubeVirt VM Fixed Address<a class=headerlink href=#kubevirt-vm-fixed-address title="Permanent link">&para;</a></h2> <p>For VM instances created by KubeVirt, <code>kube-ovn-controller</code> can assign and manage IP addresses in a similar way to the StatefulSet Pod. This allows VM instances address fixed during start-up, shutdown, upgrade, migration, and other operations throughout their lifecycle, making them more compatible with the actual virtualization user experience.</p> <p><a class=md-button href=https://ma.alauda.cn/p/0f39a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M9.5 4C5.36 4 2 6.69 2 10c0 1.89 1.08 3.56 2.78 4.66L4 17l2.5-1.5c.89.31 1.87.5 2.91.5A5.22 5.22 0 0 1 9 14c0-3.31 3.13-6 7-6 .19 0 .38 0 .56.03C15.54 5.69 12.78 4 9.5 4m-3 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m5 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1M16 9c-3.31 0-6 2.24-6 5s2.69 5 6 5c.67 0 1.31-.08 1.91-.25L20 20l-.62-1.87C20.95 17.22 22 15.71 22 14c0-2.76-2.69-5-6-5m-2 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m4 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1Z"/></svg></span> 微信群</a> <a class=md-button href=https://communityinviter.com/apps/kube-ovn/kube-ovn/ target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M6 15a2 2 0 0 1-2 2 2 2 0 0 1-2-2 2 2 0 0 1 2-2h2v2m1 0a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-5m2-8a2 2 0 0 1-2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2v2H9m0 1a2 2 0 0 1 2 2 2 2 0 0 1-2 2H4a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5m8 2a2 2 0 0 1 2-2 2 2 0 0 1 2 2 2 2 0 0 1-2 2h-2v-2m-1 0a2 2 0 0 1-2 2 2 2 0 0 1-2-2V5a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5m-2 8a2 2 0 0 1 2 2 2 2 0 0 1-2 2 2 2 0 0 1-2-2v-2h2m0-1a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-5Z"/></svg></span> Slack</a> <a class=md-button href=https://twitter.com/KubeOvn target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M22.46 6c-.77.35-1.6.58-2.46.69.88-.53 1.56-1.37 1.88-2.38-.83.5-1.75.85-2.72 1.05C18.37 4.5 17.26 4 16 4c-2.35 0-4.27 1.92-4.27 4.29 0 .34.04.67.11.98C8.28 9.09 5.11 7.38 3 4.79c-.37.63-.58 1.37-.58 2.15 0 1.49.75 2.81 1.91 3.56-.71 0-1.37-.2-1.95-.5v.03c0 2.08 1.48 3.82 3.44 4.21a4.22 4.22 0 0 1-1.93.07 4.28 4.28 0 0 0 4 2.98 8.521 8.521 0 0 1-5.33 1.84c-.34 0-.68-.02-1.02-.06C3.44 20.29 5.7 21 8.12 21 16 21 20.33 14.46 20.33 8.79c0-.19 0-.37-.01-.56.84-.6 1.56-1.36 2.14-2.23Z"/></svg></span> Twitter</a> <a class=md-button href=https://ma.alauda.cn/p/2f53a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="m20 8-8 5-8-5V6l8 5 8-5m0-2H4c-1.11 0-2 .89-2 2v12a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V6a2 2 0 0 0-2-2Z"/></svg></span> Support</a></p> <hr> <div class=md-source-file> <small> Last update: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">September 26, 2023</span> <br> Created: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">May 20, 2022</span> </small> </div> <h2 id=__comments>Comments</h2> <script src=https://giscus.app/client.js data-repo=kubeovn/kube-ovn data-repo-id="MDEwOlJlcG9zaXRvcnkxNzcwNjg5NjE=" data-category=Announcements data-category-id=DIC_kwDOCo3boc4CAj60 data-mapping=pathname data-reactions-enabled=1 data-emit-metadata=0 data-input-position=bottom data-theme=light data-lang=en crossorigin=anonymous async>
 </script> <script>
     var palette = __md_get("__palette")
     if (palette && typeof palette.color === "object")
diff --git a/v1.13.x/guide/custom-routes/index.html b/v1.13.x/guide/custom-routes/index.html
index ed952758f..0c1f23f48 100644
--- a/v1.13.x/guide/custom-routes/index.html
+++ b/v1.13.x/guide/custom-routes/index.html
@@ -17,7 +17,7 @@
 <span class=w>        </span><span class=no>&quot;gw&quot;: &quot;10.16.0.254&quot;</span>
 <span class=w>      </span><span class=no>}, {</span>
 <span class=w>        </span><span class=no>&quot;gw&quot;: &quot;10.16.0.254&quot;</span>
-<span class=w>      </span><span class=no>}]&#39;</span>
+<span class=w>      </span><span class=no>}]</span>
 <span class=nt>spec</span><span class=p>:</span>
 <span class=w>  </span><span class=nt>containers</span><span class=p>:</span>
 <span class=w>  </span><span class="p p-Indicator">-</span><span class=w> </span><span class=nt>name</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">nginx</span>
@@ -44,12 +44,12 @@
 <span class=w>            </span><span class=no>&quot;gw&quot;: &quot;10.16.0.254&quot;</span>
 <span class=w>          </span><span class=no>}, {</span>
 <span class=w>            </span><span class=no>&quot;gw&quot;: &quot;10.16.0.254&quot;</span>
-<span class=w>          </span><span class=no>}]&#39;</span>
+<span class=w>          </span><span class=no>}]</span>
 <span class=w>    </span><span class=nt>spec</span><span class=p>:</span>
 <span class=w>      </span><span class=nt>containers</span><span class=p>:</span>
 <span class=w>      </span><span class="p p-Indicator">-</span><span class=w> </span><span class=nt>name</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">nginx</span>
 <span class=w>        </span><span class=nt>image</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">docker.io/library/nginx:alpine</span>
-</code></pre></div> <p><a class=md-button href=https://ma.alauda.cn/p/0f39a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M9.5 4C5.36 4 2 6.69 2 10c0 1.89 1.08 3.56 2.78 4.66L4 17l2.5-1.5c.89.31 1.87.5 2.91.5A5.22 5.22 0 0 1 9 14c0-3.31 3.13-6 7-6 .19 0 .38 0 .56.03C15.54 5.69 12.78 4 9.5 4m-3 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m5 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1M16 9c-3.31 0-6 2.24-6 5s2.69 5 6 5c.67 0 1.31-.08 1.91-.25L20 20l-.62-1.87C20.95 17.22 22 15.71 22 14c0-2.76-2.69-5-6-5m-2 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m4 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1Z"/></svg></span> 微信群</a> <a class=md-button href=https://communityinviter.com/apps/kube-ovn/kube-ovn/ target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M6 15a2 2 0 0 1-2 2 2 2 0 0 1-2-2 2 2 0 0 1 2-2h2v2m1 0a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-5m2-8a2 2 0 0 1-2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2v2H9m0 1a2 2 0 0 1 2 2 2 2 0 0 1-2 2H4a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5m8 2a2 2 0 0 1 2-2 2 2 0 0 1 2 2 2 2 0 0 1-2 2h-2v-2m-1 0a2 2 0 0 1-2 2 2 2 0 0 1-2-2V5a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5m-2 8a2 2 0 0 1 2 2 2 2 0 0 1-2 2 2 2 0 0 1-2-2v-2h2m0-1a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-5Z"/></svg></span> Slack</a> <a class=md-button href=https://twitter.com/KubeOvn target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M22.46 6c-.77.35-1.6.58-2.46.69.88-.53 1.56-1.37 1.88-2.38-.83.5-1.75.85-2.72 1.05C18.37 4.5 17.26 4 16 4c-2.35 0-4.27 1.92-4.27 4.29 0 .34.04.67.11.98C8.28 9.09 5.11 7.38 3 4.79c-.37.63-.58 1.37-.58 2.15 0 1.49.75 2.81 1.91 3.56-.71 0-1.37-.2-1.95-.5v.03c0 2.08 1.48 3.82 3.44 4.21a4.22 4.22 0 0 1-1.93.07 4.28 4.28 0 0 0 4 2.98 8.521 8.521 0 0 1-5.33 1.84c-.34 0-.68-.02-1.02-.06C3.44 20.29 5.7 21 8.12 21 16 21 20.33 14.46 20.33 8.79c0-.19 0-.37-.01-.56.84-.6 1.56-1.36 2.14-2.23Z"/></svg></span> Twitter</a> <a class=md-button href=https://ma.alauda.cn/p/2f53a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="m20 8-8 5-8-5V6l8 5 8-5m0-2H4c-1.11 0-2 .89-2 2v12a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V6a2 2 0 0 0-2-2Z"/></svg></span> Support</a></p> <hr> <div class=md-source-file> <small> 最后更新: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2023年5月9日</span> <br> 创建日期: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2023年2月16日</span> </small> </div> <h2 id=__comments>评论</h2> <script src=https://giscus.app/client.js data-repo=kubeovn/kube-ovn data-repo-id="MDEwOlJlcG9zaXRvcnkxNzcwNjg5NjE=" data-category=Announcements data-category-id=DIC_kwDOCo3boc4CAj60 data-mapping=pathname data-reactions-enabled=1 data-emit-metadata=0 data-input-position=bottom data-theme=light data-lang=zh-CN crossorigin=anonymous async>
+</code></pre></div> <p><a class=md-button href=https://ma.alauda.cn/p/0f39a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M9.5 4C5.36 4 2 6.69 2 10c0 1.89 1.08 3.56 2.78 4.66L4 17l2.5-1.5c.89.31 1.87.5 2.91.5A5.22 5.22 0 0 1 9 14c0-3.31 3.13-6 7-6 .19 0 .38 0 .56.03C15.54 5.69 12.78 4 9.5 4m-3 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m5 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1M16 9c-3.31 0-6 2.24-6 5s2.69 5 6 5c.67 0 1.31-.08 1.91-.25L20 20l-.62-1.87C20.95 17.22 22 15.71 22 14c0-2.76-2.69-5-6-5m-2 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m4 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1Z"/></svg></span> 微信群</a> <a class=md-button href=https://communityinviter.com/apps/kube-ovn/kube-ovn/ target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M6 15a2 2 0 0 1-2 2 2 2 0 0 1-2-2 2 2 0 0 1 2-2h2v2m1 0a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-5m2-8a2 2 0 0 1-2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2v2H9m0 1a2 2 0 0 1 2 2 2 2 0 0 1-2 2H4a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5m8 2a2 2 0 0 1 2-2 2 2 0 0 1 2 2 2 2 0 0 1-2 2h-2v-2m-1 0a2 2 0 0 1-2 2 2 2 0 0 1-2-2V5a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5m-2 8a2 2 0 0 1 2 2 2 2 0 0 1-2 2 2 2 0 0 1-2-2v-2h2m0-1a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-5Z"/></svg></span> Slack</a> <a class=md-button href=https://twitter.com/KubeOvn target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M22.46 6c-.77.35-1.6.58-2.46.69.88-.53 1.56-1.37 1.88-2.38-.83.5-1.75.85-2.72 1.05C18.37 4.5 17.26 4 16 4c-2.35 0-4.27 1.92-4.27 4.29 0 .34.04.67.11.98C8.28 9.09 5.11 7.38 3 4.79c-.37.63-.58 1.37-.58 2.15 0 1.49.75 2.81 1.91 3.56-.71 0-1.37-.2-1.95-.5v.03c0 2.08 1.48 3.82 3.44 4.21a4.22 4.22 0 0 1-1.93.07 4.28 4.28 0 0 0 4 2.98 8.521 8.521 0 0 1-5.33 1.84c-.34 0-.68-.02-1.02-.06C3.44 20.29 5.7 21 8.12 21 16 21 20.33 14.46 20.33 8.79c0-.19 0-.37-.01-.56.84-.6 1.56-1.36 2.14-2.23Z"/></svg></span> Twitter</a> <a class=md-button href=https://ma.alauda.cn/p/2f53a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="m20 8-8 5-8-5V6l8 5 8-5m0-2H4c-1.11 0-2 .89-2 2v12a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V6a2 2 0 0 0-2-2Z"/></svg></span> Support</a></p> <hr> <div class=md-source-file> <small> 最后更新: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2023年9月26日</span> <br> 创建日期: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2023年2月16日</span> </small> </div> <h2 id=__comments>评论</h2> <script src=https://giscus.app/client.js data-repo=kubeovn/kube-ovn data-repo-id="MDEwOlJlcG9zaXRvcnkxNzcwNjg5NjE=" data-category=Announcements data-category-id=DIC_kwDOCo3boc4CAj60 data-mapping=pathname data-reactions-enabled=1 data-emit-metadata=0 data-input-position=bottom data-theme=light data-lang=zh-CN crossorigin=anonymous async>
 </script> <script>
     var palette = __md_get("__palette")
     if (palette && typeof palette.color === "object")
diff --git a/v1.13.x/guide/static-ip-mac/index.html b/v1.13.x/guide/static-ip-mac/index.html
index 1b4fc1db7..db8f5ac3d 100644
--- a/v1.13.x/guide/static-ip-mac/index.html
+++ b/v1.13.x/guide/static-ip-mac/index.html
@@ -17,7 +17,7 @@
 <span class=w>  </span><span class=nt>containers</span><span class=p>:</span>
 <span class=w>  </span><span class="p p-Indicator">-</span><span class=w> </span><span class=nt>name</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">static-ip</span>
 <span class=w>    </span><span class=nt>image</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">docker.io/library/nginx:alpine</span>
-</code></pre></div> <p>在使用 annotation 定义单个 Pod IP/Mac 时需要注意以下几点：</p> <ol> <li>所使用的 IP/Mac 不能和已有的 IP/Mac 冲突。</li> <li>IP 必须在所属子网的 CIDR 内。</li> <li>可以只指定 IP 或 Mac，只指定一个时，另一个会随机分配。</li> </ol> <h2 id=workload-ip-pool>Workload 通用 IP Pool 固定地址<a class=headerlink href=#workload-ip-pool title="Permanent link">&para;</a></h2> <p>Kube-OVN 支持通过 annotation <code>ovn.kubernetes.io/ip_pool</code> 给 Workload（Deployment/StatefulSet/DaemonSet/Job/CronJob）设置固定 IP。 <code>kube-ovn-controllerr</code> 会自动选择 <code>ovn.kubernetes.io/ip_pool</code> 中指定的 IP 并进行冲突检测。</p> <p>IP Pool 的 Annotation 需要加在 <code>template</code> 内的 <code>annotation</code> 字段，除了 Kubernetes 内置的 Workload 类型， 其他用户自定义的 Workload 也可以使用同样的方式进行固定地址分配。</p> <h3 id=deployment-ip>Deployment 固定 IP 示例<a class=headerlink href=#deployment-ip title="Permanent link">&para;</a></h3> <div class=highlight><pre><span></span><code><span class=nt>apiVersion</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">apps/v1</span>
+</code></pre></div> <p>在使用 annotation 定义单个 Pod IP/Mac 时需要注意以下几点：</p> <ol> <li>所使用的 IP/Mac 不能和已有的 IP/Mac 冲突。</li> <li>IP 必须在所属子网的 CIDR 内。</li> <li>可以只指定 IP 或 Mac，只指定一个时，另一个会随机分配。</li> </ol> <h2 id=workload-ip-pool>Workload 通用 IP Pool 固定地址<a class=headerlink href=#workload-ip-pool title="Permanent link">&para;</a></h2> <p>Kube-OVN 支持通过 annotation <code>ovn.kubernetes.io/ip_pool</code> 给 Workload（Deployment/StatefulSet/DaemonSet/Job/CronJob）设置固定 IP。 <code>kube-ovn-controller</code> 会自动选择 <code>ovn.kubernetes.io/ip_pool</code> 中指定的 IP 并进行冲突检测。</p> <p>IP Pool 的 Annotation 需要加在 <code>template</code> 内的 <code>annotation</code> 字段，除了 Kubernetes 内置的 Workload 类型， 其他用户自定义的 Workload 也可以使用同样的方式进行固定地址分配。</p> <h3 id=deployment-ip>Deployment 固定 IP 示例<a class=headerlink href=#deployment-ip title="Permanent link">&para;</a></h3> <div class=highlight><pre><span></span><code><span class=nt>apiVersion</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">apps/v1</span>
 <span class=nt>kind</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">Deployment</span>
 <span class=nt>metadata</span><span class=p>:</span>
 <span class=w>  </span><span class=nt>name</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">ippool</span>
@@ -59,7 +59,7 @@
 <span class=w>        </span><span class=nt>ports</span><span class=p>:</span>
 <span class=w>        </span><span class="p p-Indicator">-</span><span class=w> </span><span class=nt>containerPort</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">80</span>
 <span class=w>          </span><span class=nt>name</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">web</span>
-</code></pre></div> <p>可以尝试删除 StatefulSet 下 Pod 观察 Pod IP 变化信息。</p> <h2 id=kubevirt-vm>KubeVirt VM 固定地址<a class=headerlink href=#kubevirt-vm title="Permanent link">&para;</a></h2> <p>针对 KubeVirt 创建的 VM 实例，<code>kube-ovn-controller</code> 可以按照类似 StatefulSet Pod 的方式进行 IP 地址分配和管理。 以达到 VM 实例在生命周期内启停，升级，迁移等操作过程中地址固定不变，更符虚拟化合用户的实际使用体验。</p> <p><a class=md-button href=https://ma.alauda.cn/p/0f39a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M9.5 4C5.36 4 2 6.69 2 10c0 1.89 1.08 3.56 2.78 4.66L4 17l2.5-1.5c.89.31 1.87.5 2.91.5A5.22 5.22 0 0 1 9 14c0-3.31 3.13-6 7-6 .19 0 .38 0 .56.03C15.54 5.69 12.78 4 9.5 4m-3 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m5 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1M16 9c-3.31 0-6 2.24-6 5s2.69 5 6 5c.67 0 1.31-.08 1.91-.25L20 20l-.62-1.87C20.95 17.22 22 15.71 22 14c0-2.76-2.69-5-6-5m-2 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m4 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1Z"/></svg></span> 微信群</a> <a class=md-button href=https://communityinviter.com/apps/kube-ovn/kube-ovn/ target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M6 15a2 2 0 0 1-2 2 2 2 0 0 1-2-2 2 2 0 0 1 2-2h2v2m1 0a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-5m2-8a2 2 0 0 1-2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2v2H9m0 1a2 2 0 0 1 2 2 2 2 0 0 1-2 2H4a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5m8 2a2 2 0 0 1 2-2 2 2 0 0 1 2 2 2 2 0 0 1-2 2h-2v-2m-1 0a2 2 0 0 1-2 2 2 2 0 0 1-2-2V5a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5m-2 8a2 2 0 0 1 2 2 2 2 0 0 1-2 2 2 2 0 0 1-2-2v-2h2m0-1a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-5Z"/></svg></span> Slack</a> <a class=md-button href=https://twitter.com/KubeOvn target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M22.46 6c-.77.35-1.6.58-2.46.69.88-.53 1.56-1.37 1.88-2.38-.83.5-1.75.85-2.72 1.05C18.37 4.5 17.26 4 16 4c-2.35 0-4.27 1.92-4.27 4.29 0 .34.04.67.11.98C8.28 9.09 5.11 7.38 3 4.79c-.37.63-.58 1.37-.58 2.15 0 1.49.75 2.81 1.91 3.56-.71 0-1.37-.2-1.95-.5v.03c0 2.08 1.48 3.82 3.44 4.21a4.22 4.22 0 0 1-1.93.07 4.28 4.28 0 0 0 4 2.98 8.521 8.521 0 0 1-5.33 1.84c-.34 0-.68-.02-1.02-.06C3.44 20.29 5.7 21 8.12 21 16 21 20.33 14.46 20.33 8.79c0-.19 0-.37-.01-.56.84-.6 1.56-1.36 2.14-2.23Z"/></svg></span> Twitter</a> <a class=md-button href=https://ma.alauda.cn/p/2f53a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="m20 8-8 5-8-5V6l8 5 8-5m0-2H4c-1.11 0-2 .89-2 2v12a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V6a2 2 0 0 0-2-2Z"/></svg></span> Support</a></p> <hr> <div class=md-source-file> <small> 最后更新: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2023年9月12日</span> <br> 创建日期: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2022年5月20日</span> </small> </div> <h2 id=__comments>评论</h2> <script src=https://giscus.app/client.js data-repo=kubeovn/kube-ovn data-repo-id="MDEwOlJlcG9zaXRvcnkxNzcwNjg5NjE=" data-category=Announcements data-category-id=DIC_kwDOCo3boc4CAj60 data-mapping=pathname data-reactions-enabled=1 data-emit-metadata=0 data-input-position=bottom data-theme=light data-lang=zh-CN crossorigin=anonymous async>
+</code></pre></div> <p>可以尝试删除 StatefulSet 下 Pod 观察 Pod IP 变化信息。</p> <h2 id=kubevirt-vm>KubeVirt VM 固定地址<a class=headerlink href=#kubevirt-vm title="Permanent link">&para;</a></h2> <p>针对 KubeVirt 创建的 VM 实例，<code>kube-ovn-controller</code> 可以按照类似 StatefulSet Pod 的方式进行 IP 地址分配和管理。 以达到 VM 实例在生命周期内启停，升级，迁移等操作过程中地址固定不变，更符虚拟化合用户的实际使用体验。</p> <p><a class=md-button href=https://ma.alauda.cn/p/0f39a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M9.5 4C5.36 4 2 6.69 2 10c0 1.89 1.08 3.56 2.78 4.66L4 17l2.5-1.5c.89.31 1.87.5 2.91.5A5.22 5.22 0 0 1 9 14c0-3.31 3.13-6 7-6 .19 0 .38 0 .56.03C15.54 5.69 12.78 4 9.5 4m-3 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m5 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1M16 9c-3.31 0-6 2.24-6 5s2.69 5 6 5c.67 0 1.31-.08 1.91-.25L20 20l-.62-1.87C20.95 17.22 22 15.71 22 14c0-2.76-2.69-5-6-5m-2 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m4 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1Z"/></svg></span> 微信群</a> <a class=md-button href=https://communityinviter.com/apps/kube-ovn/kube-ovn/ target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M6 15a2 2 0 0 1-2 2 2 2 0 0 1-2-2 2 2 0 0 1 2-2h2v2m1 0a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-5m2-8a2 2 0 0 1-2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2v2H9m0 1a2 2 0 0 1 2 2 2 2 0 0 1-2 2H4a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5m8 2a2 2 0 0 1 2-2 2 2 0 0 1 2 2 2 2 0 0 1-2 2h-2v-2m-1 0a2 2 0 0 1-2 2 2 2 0 0 1-2-2V5a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5m-2 8a2 2 0 0 1 2 2 2 2 0 0 1-2 2 2 2 0 0 1-2-2v-2h2m0-1a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-5Z"/></svg></span> Slack</a> <a class=md-button href=https://twitter.com/KubeOvn target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M22.46 6c-.77.35-1.6.58-2.46.69.88-.53 1.56-1.37 1.88-2.38-.83.5-1.75.85-2.72 1.05C18.37 4.5 17.26 4 16 4c-2.35 0-4.27 1.92-4.27 4.29 0 .34.04.67.11.98C8.28 9.09 5.11 7.38 3 4.79c-.37.63-.58 1.37-.58 2.15 0 1.49.75 2.81 1.91 3.56-.71 0-1.37-.2-1.95-.5v.03c0 2.08 1.48 3.82 3.44 4.21a4.22 4.22 0 0 1-1.93.07 4.28 4.28 0 0 0 4 2.98 8.521 8.521 0 0 1-5.33 1.84c-.34 0-.68-.02-1.02-.06C3.44 20.29 5.7 21 8.12 21 16 21 20.33 14.46 20.33 8.79c0-.19 0-.37-.01-.56.84-.6 1.56-1.36 2.14-2.23Z"/></svg></span> Twitter</a> <a class=md-button href=https://ma.alauda.cn/p/2f53a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="m20 8-8 5-8-5V6l8 5 8-5m0-2H4c-1.11 0-2 .89-2 2v12a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V6a2 2 0 0 0-2-2Z"/></svg></span> Support</a></p> <hr> <div class=md-source-file> <small> 最后更新: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2023年9月26日</span> <br> 创建日期: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2022年5月20日</span> </small> </div> <h2 id=__comments>评论</h2> <script src=https://giscus.app/client.js data-repo=kubeovn/kube-ovn data-repo-id="MDEwOlJlcG9zaXRvcnkxNzcwNjg5NjE=" data-category=Announcements data-category-id=DIC_kwDOCo3boc4CAj60 data-mapping=pathname data-reactions-enabled=1 data-emit-metadata=0 data-input-position=bottom data-theme=light data-lang=zh-CN crossorigin=anonymous async>
 </script> <script>
     var palette = __md_get("__palette")
     if (palette && typeof palette.color === "object")
diff --git a/v1.13.x/guide/subnet/index.html b/v1.13.x/guide/subnet/index.html
index abbcf7beb..bb8ac29cd 100644
--- a/v1.13.x/guide/subnet/index.html
+++ b/v1.13.x/guide/subnet/index.html
@@ -147,13 +147,13 @@
 <span class=w>  </span><span class=nt>name</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">disable-gw-check</span>
 <span class=nt>spec</span><span class=p>:</span>
 <span class=w>  </span><span class=nt>disableGatewayCheck</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
-</code></pre></div> <h2 id=multicast-snoop>Multicast-Snoop 配置<a class=headerlink href=#multicast-snoop title="Permanent link">&para;</a></h2> <p>默认情况下 subnet 下的 Pod 如果发送组播报文，OVN 的默认行为是会广播组播报文到子网下所有的 Pod。如果开启subnet 的 multicast snoop 开关，OVN 会根据 <code>South Database</code> 中的组播表 <code>Multicast_Group</code> 来进行转发，而不在进行广播。</p> <div class=highlight><pre><span></span><code><span class=nt>apiVersion</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">kubeovn.io/v1</span>
+</code></pre></div> <h2 id=multicast-snoop>Multicast-Snoop 配置<a class=headerlink href=#multicast-snoop title="Permanent link">&para;</a></h2> <p>默认情况下 subnet 下的 Pod 如果发送组播报文，OVN 的默认行为是会广播组播报文到子网下所有的 Pod。如果开启 subnet 的 multicast snoop 开关，OVN 会根据 <code>South Database</code> 中的组播表 <code>Multicast_Group</code> 来进行转发，而不在进行广播。</p> <div class=highlight><pre><span></span><code><span class=nt>apiVersion</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">kubeovn.io/v1</span>
 <span class=nt>kind</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">Subnet</span>
 <span class=nt>metadata</span><span class=p>:</span>
 <span class=w>  </span><span class=nt>name</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">sample1</span>
 <span class=nt>spec</span><span class=p>:</span>
 <span class=w>  </span><span class=nt>enableMulticastSnoop</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
-</code></pre></div> <h2 id=_12>其他高级设置<a class=headerlink href=#_12 title="Permanent link">&para;</a></h2> <ul> <li><a href=../ippool/ >IP 池使用</a></li> <li><a href=../../advance/nat-policy-rule/ >默认 VPC NAT 策略规则</a></li> <li><a href=../qos/ >QoS 设置</a></li> <li><a href=../../advance/multi-nic/ >多网卡管理</a></li> <li><a href=../../advance/dhcp/ >DHCP 选项</a></li> <li><a href=../../advance/external-gateway/ >外部网关设置</a></li> <li><a href=../../advance/with-ovn-ic/ >集群互联设置</a></li> <li><a href=../../advance/vip/ >虚拟 IP 设置</a></li> </ul> <p><a class=md-button href=https://ma.alauda.cn/p/0f39a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M9.5 4C5.36 4 2 6.69 2 10c0 1.89 1.08 3.56 2.78 4.66L4 17l2.5-1.5c.89.31 1.87.5 2.91.5A5.22 5.22 0 0 1 9 14c0-3.31 3.13-6 7-6 .19 0 .38 0 .56.03C15.54 5.69 12.78 4 9.5 4m-3 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m5 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1M16 9c-3.31 0-6 2.24-6 5s2.69 5 6 5c.67 0 1.31-.08 1.91-.25L20 20l-.62-1.87C20.95 17.22 22 15.71 22 14c0-2.76-2.69-5-6-5m-2 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m4 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1Z"/></svg></span> 微信群</a> <a class=md-button href=https://communityinviter.com/apps/kube-ovn/kube-ovn/ target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M6 15a2 2 0 0 1-2 2 2 2 0 0 1-2-2 2 2 0 0 1 2-2h2v2m1 0a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-5m2-8a2 2 0 0 1-2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2v2H9m0 1a2 2 0 0 1 2 2 2 2 0 0 1-2 2H4a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5m8 2a2 2 0 0 1 2-2 2 2 0 0 1 2 2 2 2 0 0 1-2 2h-2v-2m-1 0a2 2 0 0 1-2 2 2 2 0 0 1-2-2V5a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5m-2 8a2 2 0 0 1 2 2 2 2 0 0 1-2 2 2 2 0 0 1-2-2v-2h2m0-1a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-5Z"/></svg></span> Slack</a> <a class=md-button href=https://twitter.com/KubeOvn target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M22.46 6c-.77.35-1.6.58-2.46.69.88-.53 1.56-1.37 1.88-2.38-.83.5-1.75.85-2.72 1.05C18.37 4.5 17.26 4 16 4c-2.35 0-4.27 1.92-4.27 4.29 0 .34.04.67.11.98C8.28 9.09 5.11 7.38 3 4.79c-.37.63-.58 1.37-.58 2.15 0 1.49.75 2.81 1.91 3.56-.71 0-1.37-.2-1.95-.5v.03c0 2.08 1.48 3.82 3.44 4.21a4.22 4.22 0 0 1-1.93.07 4.28 4.28 0 0 0 4 2.98 8.521 8.521 0 0 1-5.33 1.84c-.34 0-.68-.02-1.02-.06C3.44 20.29 5.7 21 8.12 21 16 21 20.33 14.46 20.33 8.79c0-.19 0-.37-.01-.56.84-.6 1.56-1.36 2.14-2.23Z"/></svg></span> Twitter</a> <a class=md-button href=https://ma.alauda.cn/p/2f53a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="m20 8-8 5-8-5V6l8 5 8-5m0-2H4c-1.11 0-2 .89-2 2v12a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V6a2 2 0 0 0-2-2Z"/></svg></span> Support</a></p> <hr> <div class=md-source-file> <small> 最后更新: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2023年9月19日</span> <br> 创建日期: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2022年5月24日</span> </small> </div> <h2 id=__comments>评论</h2> <script src=https://giscus.app/client.js data-repo=kubeovn/kube-ovn data-repo-id="MDEwOlJlcG9zaXRvcnkxNzcwNjg5NjE=" data-category=Announcements data-category-id=DIC_kwDOCo3boc4CAj60 data-mapping=pathname data-reactions-enabled=1 data-emit-metadata=0 data-input-position=bottom data-theme=light data-lang=zh-CN crossorigin=anonymous async>
+</code></pre></div> <h2 id=_12>其他高级设置<a class=headerlink href=#_12 title="Permanent link">&para;</a></h2> <ul> <li><a href=../ippool/ >IP 池使用</a></li> <li><a href=../../advance/nat-policy-rule/ >默认 VPC NAT 策略规则</a></li> <li><a href=../qos/ >QoS 设置</a></li> <li><a href=../../advance/multi-nic/ >多网卡管理</a></li> <li><a href=../../advance/dhcp/ >DHCP 选项</a></li> <li><a href=../../advance/external-gateway/ >外部网关设置</a></li> <li><a href=../../advance/with-ovn-ic/ >集群互联设置</a></li> <li><a href=../../advance/vip/ >虚拟 IP 设置</a></li> </ul> <p><a class=md-button href=https://ma.alauda.cn/p/0f39a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M9.5 4C5.36 4 2 6.69 2 10c0 1.89 1.08 3.56 2.78 4.66L4 17l2.5-1.5c.89.31 1.87.5 2.91.5A5.22 5.22 0 0 1 9 14c0-3.31 3.13-6 7-6 .19 0 .38 0 .56.03C15.54 5.69 12.78 4 9.5 4m-3 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m5 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1M16 9c-3.31 0-6 2.24-6 5s2.69 5 6 5c.67 0 1.31-.08 1.91-.25L20 20l-.62-1.87C20.95 17.22 22 15.71 22 14c0-2.76-2.69-5-6-5m-2 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m4 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1Z"/></svg></span> 微信群</a> <a class=md-button href=https://communityinviter.com/apps/kube-ovn/kube-ovn/ target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M6 15a2 2 0 0 1-2 2 2 2 0 0 1-2-2 2 2 0 0 1 2-2h2v2m1 0a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-5m2-8a2 2 0 0 1-2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2v2H9m0 1a2 2 0 0 1 2 2 2 2 0 0 1-2 2H4a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5m8 2a2 2 0 0 1 2-2 2 2 0 0 1 2 2 2 2 0 0 1-2 2h-2v-2m-1 0a2 2 0 0 1-2 2 2 2 0 0 1-2-2V5a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5m-2 8a2 2 0 0 1 2 2 2 2 0 0 1-2 2 2 2 0 0 1-2-2v-2h2m0-1a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-5Z"/></svg></span> Slack</a> <a class=md-button href=https://twitter.com/KubeOvn target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M22.46 6c-.77.35-1.6.58-2.46.69.88-.53 1.56-1.37 1.88-2.38-.83.5-1.75.85-2.72 1.05C18.37 4.5 17.26 4 16 4c-2.35 0-4.27 1.92-4.27 4.29 0 .34.04.67.11.98C8.28 9.09 5.11 7.38 3 4.79c-.37.63-.58 1.37-.58 2.15 0 1.49.75 2.81 1.91 3.56-.71 0-1.37-.2-1.95-.5v.03c0 2.08 1.48 3.82 3.44 4.21a4.22 4.22 0 0 1-1.93.07 4.28 4.28 0 0 0 4 2.98 8.521 8.521 0 0 1-5.33 1.84c-.34 0-.68-.02-1.02-.06C3.44 20.29 5.7 21 8.12 21 16 21 20.33 14.46 20.33 8.79c0-.19 0-.37-.01-.56.84-.6 1.56-1.36 2.14-2.23Z"/></svg></span> Twitter</a> <a class=md-button href=https://ma.alauda.cn/p/2f53a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="m20 8-8 5-8-5V6l8 5 8-5m0-2H4c-1.11 0-2 .89-2 2v12a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V6a2 2 0 0 0-2-2Z"/></svg></span> Support</a></p> <hr> <div class=md-source-file> <small> 最后更新: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2023年9月26日</span> <br> 创建日期: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2022年5月24日</span> </small> </div> <h2 id=__comments>评论</h2> <script src=https://giscus.app/client.js data-repo=kubeovn/kube-ovn data-repo-id="MDEwOlJlcG9zaXRvcnkxNzcwNjg5NjE=" data-category=Announcements data-category-id=DIC_kwDOCo3boc4CAj60 data-mapping=pathname data-reactions-enabled=1 data-emit-metadata=0 data-input-position=bottom data-theme=light data-lang=zh-CN crossorigin=anonymous async>
 </script> <script>
     var palette = __md_get("__palette")
     if (palette && typeof palette.color === "object")
diff --git a/v1.13.x/guide/vpc/index.html b/v1.13.x/guide/vpc/index.html
index 03abdf4c0..4d950cff3 100644
--- a/v1.13.x/guide/vpc/index.html
+++ b/v1.13.x/guide/vpc/index.html
@@ -182,7 +182,7 @@
 <span class=nt>spec</span><span class=p>:</span>
 <span class=w>  </span><span class=nt>eip</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">eips01</span>
 <span class=w>  </span><span class=nt>internalCIDR</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">10.0.1.0/24</span>
-</code></pre></div> <h3 id=ip>创建浮动 IP<a class=headerlink href=#ip title="Permanent link">&para;</a></h3> <p>通过浮动 IP 规则，VPC 内的一个 IP 会和 EIP 进行完全映射，外部可以通过这个 EIP 方位 VPC 内的 IP，VPC 内的这个 IP 访问外部地址时也会 SNAT 成这个 EIP。</p> <div class=highlight><pre><span></span><code><span class=nn>---</span>
+</code></pre></div> <h3 id=ip>创建浮动 IP<a class=headerlink href=#ip title="Permanent link">&para;</a></h3> <p>通过浮动 IP 规则，VPC 内的一个 IP 会和 EIP 进行完全映射，外部可以通过这个 EIP 访问 VPC 内的 IP，VPC 内的这个 IP 访问外部地址时也会 SNAT 成这个 EIP。</p> <div class=highlight><pre><span></span><code><span class=nn>---</span>
 <span class=nt>kind</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">IptablesEIP</span>
 <span class=nt>apiVersion</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">kubeovn.io/v1</span>
 <span class=nt>metadata</span><span class=p>:</span>
@@ -370,7 +370,7 @@
 NAME<span class=w>        </span>ACTIVE<span class=w>   </span>VPC<span class=w>         </span>SUBNET<span class=w>   </span>
 test-cjh1<span class=w>   </span><span class=nb>false</span><span class=w>    </span>cjh-vpc-1<span class=w>   </span>cjh-subnet-1<span class=w>   </span>
 test-cjh2<span class=w>   </span><span class=nb>true</span><span class=w>     </span>cjh-vpc-1<span class=w>   </span>cjh-subnet-2<span class=w> </span>
-</code></pre></div> <ul> <li><code>ACTIVE</code>: <code>true</code> 成功部署了自定义 dns 组件，<code>false</code> 无部署</li> </ul> <h3 id=_7>限制<a class=headerlink href=#_7 title="Permanent link">&para;</a></h3> <ul> <li>一个 vpc 下只会部署一个自定义 dns 组件;</li> <li>当一个 vpc 下配置多个 vpc-dns 资源（即同一个 vpc 不同的 subnet），只有一个 vpc-dns 资源状态 <code>true</code>，其他为 <code>fasle</code>;</li> <li>当 <code>true</code> 的 vpc-dns 被删除掉，会获取其他 <code>false</code> 的 vpc-dns 进行部署。</li> </ul> <p><a class=md-button href=https://ma.alauda.cn/p/0f39a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M9.5 4C5.36 4 2 6.69 2 10c0 1.89 1.08 3.56 2.78 4.66L4 17l2.5-1.5c.89.31 1.87.5 2.91.5A5.22 5.22 0 0 1 9 14c0-3.31 3.13-6 7-6 .19 0 .38 0 .56.03C15.54 5.69 12.78 4 9.5 4m-3 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m5 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1M16 9c-3.31 0-6 2.24-6 5s2.69 5 6 5c.67 0 1.31-.08 1.91-.25L20 20l-.62-1.87C20.95 17.22 22 15.71 22 14c0-2.76-2.69-5-6-5m-2 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m4 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1Z"/></svg></span> 微信群</a> <a class=md-button href=https://communityinviter.com/apps/kube-ovn/kube-ovn/ target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M6 15a2 2 0 0 1-2 2 2 2 0 0 1-2-2 2 2 0 0 1 2-2h2v2m1 0a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-5m2-8a2 2 0 0 1-2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2v2H9m0 1a2 2 0 0 1 2 2 2 2 0 0 1-2 2H4a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5m8 2a2 2 0 0 1 2-2 2 2 0 0 1 2 2 2 2 0 0 1-2 2h-2v-2m-1 0a2 2 0 0 1-2 2 2 2 0 0 1-2-2V5a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5m-2 8a2 2 0 0 1 2 2 2 2 0 0 1-2 2 2 2 0 0 1-2-2v-2h2m0-1a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-5Z"/></svg></span> Slack</a> <a class=md-button href=https://twitter.com/KubeOvn target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M22.46 6c-.77.35-1.6.58-2.46.69.88-.53 1.56-1.37 1.88-2.38-.83.5-1.75.85-2.72 1.05C18.37 4.5 17.26 4 16 4c-2.35 0-4.27 1.92-4.27 4.29 0 .34.04.67.11.98C8.28 9.09 5.11 7.38 3 4.79c-.37.63-.58 1.37-.58 2.15 0 1.49.75 2.81 1.91 3.56-.71 0-1.37-.2-1.95-.5v.03c0 2.08 1.48 3.82 3.44 4.21a4.22 4.22 0 0 1-1.93.07 4.28 4.28 0 0 0 4 2.98 8.521 8.521 0 0 1-5.33 1.84c-.34 0-.68-.02-1.02-.06C3.44 20.29 5.7 21 8.12 21 16 21 20.33 14.46 20.33 8.79c0-.19 0-.37-.01-.56.84-.6 1.56-1.36 2.14-2.23Z"/></svg></span> Twitter</a> <a class=md-button href=https://ma.alauda.cn/p/2f53a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="m20 8-8 5-8-5V6l8 5 8-5m0-2H4c-1.11 0-2 .89-2 2v12a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V6a2 2 0 0 0-2-2Z"/></svg></span> Support</a></p> <hr> <div class=md-source-file> <small> 最后更新: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2023年8月17日</span> <br> 创建日期: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2022年5月24日</span> </small> </div> <h2 id=__comments>评论</h2> <script src=https://giscus.app/client.js data-repo=kubeovn/kube-ovn data-repo-id="MDEwOlJlcG9zaXRvcnkxNzcwNjg5NjE=" data-category=Announcements data-category-id=DIC_kwDOCo3boc4CAj60 data-mapping=pathname data-reactions-enabled=1 data-emit-metadata=0 data-input-position=bottom data-theme=light data-lang=zh-CN crossorigin=anonymous async>
+</code></pre></div> <ul> <li><code>ACTIVE</code>: <code>true</code> 成功部署了自定义 dns 组件，<code>false</code> 无部署</li> </ul> <h3 id=_7>限制<a class=headerlink href=#_7 title="Permanent link">&para;</a></h3> <ul> <li>一个 vpc 下只会部署一个自定义 dns 组件;</li> <li>当一个 vpc 下配置多个 vpc-dns 资源（即同一个 vpc 不同的 subnet），只有一个 vpc-dns 资源状态 <code>true</code>，其他为 <code>fasle</code>;</li> <li>当 <code>true</code> 的 vpc-dns 被删除掉，会获取其他 <code>false</code> 的 vpc-dns 进行部署。</li> </ul> <p><a class=md-button href=https://ma.alauda.cn/p/0f39a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M9.5 4C5.36 4 2 6.69 2 10c0 1.89 1.08 3.56 2.78 4.66L4 17l2.5-1.5c.89.31 1.87.5 2.91.5A5.22 5.22 0 0 1 9 14c0-3.31 3.13-6 7-6 .19 0 .38 0 .56.03C15.54 5.69 12.78 4 9.5 4m-3 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m5 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1M16 9c-3.31 0-6 2.24-6 5s2.69 5 6 5c.67 0 1.31-.08 1.91-.25L20 20l-.62-1.87C20.95 17.22 22 15.71 22 14c0-2.76-2.69-5-6-5m-2 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m4 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1Z"/></svg></span> 微信群</a> <a class=md-button href=https://communityinviter.com/apps/kube-ovn/kube-ovn/ target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M6 15a2 2 0 0 1-2 2 2 2 0 0 1-2-2 2 2 0 0 1 2-2h2v2m1 0a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-5m2-8a2 2 0 0 1-2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2v2H9m0 1a2 2 0 0 1 2 2 2 2 0 0 1-2 2H4a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5m8 2a2 2 0 0 1 2-2 2 2 0 0 1 2 2 2 2 0 0 1-2 2h-2v-2m-1 0a2 2 0 0 1-2 2 2 2 0 0 1-2-2V5a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5m-2 8a2 2 0 0 1 2 2 2 2 0 0 1-2 2 2 2 0 0 1-2-2v-2h2m0-1a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-5Z"/></svg></span> Slack</a> <a class=md-button href=https://twitter.com/KubeOvn target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M22.46 6c-.77.35-1.6.58-2.46.69.88-.53 1.56-1.37 1.88-2.38-.83.5-1.75.85-2.72 1.05C18.37 4.5 17.26 4 16 4c-2.35 0-4.27 1.92-4.27 4.29 0 .34.04.67.11.98C8.28 9.09 5.11 7.38 3 4.79c-.37.63-.58 1.37-.58 2.15 0 1.49.75 2.81 1.91 3.56-.71 0-1.37-.2-1.95-.5v.03c0 2.08 1.48 3.82 3.44 4.21a4.22 4.22 0 0 1-1.93.07 4.28 4.28 0 0 0 4 2.98 8.521 8.521 0 0 1-5.33 1.84c-.34 0-.68-.02-1.02-.06C3.44 20.29 5.7 21 8.12 21 16 21 20.33 14.46 20.33 8.79c0-.19 0-.37-.01-.56.84-.6 1.56-1.36 2.14-2.23Z"/></svg></span> Twitter</a> <a class=md-button href=https://ma.alauda.cn/p/2f53a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="m20 8-8 5-8-5V6l8 5 8-5m0-2H4c-1.11 0-2 .89-2 2v12a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V6a2 2 0 0 0-2-2Z"/></svg></span> Support</a></p> <hr> <div class=md-source-file> <small> 最后更新: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2023年9月26日</span> <br> 创建日期: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2022年5月24日</span> </small> </div> <h2 id=__comments>评论</h2> <script src=https://giscus.app/client.js data-repo=kubeovn/kube-ovn data-repo-id="MDEwOlJlcG9zaXRvcnkxNzcwNjg5NjE=" data-category=Announcements data-category-id=DIC_kwDOCo3boc4CAj60 data-mapping=pathname data-reactions-enabled=1 data-emit-metadata=0 data-input-position=bottom data-theme=light data-lang=zh-CN crossorigin=anonymous async>
 </script> <script>
     var palette = __md_get("__palette")
     if (palette && typeof palette.color === "object")
diff --git a/v1.13.x/search/search_index.json b/v1.13.x/search/search_index.json
index cb79da633..3492c3bfe 100644
--- a/v1.13.x/search/search_index.json
+++ b/v1.13.x/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["ja","en"],"separator":"[\\s\\u200b\\-]","pipeline":["stemmer"]},"docs":[{"location":"","title":"Kube-OVN","text":"<p>Kube-OVN \u662f\u4e00\u6b3e CNCF \u65d7\u4e0b\u7684\u4f01\u4e1a\u7ea7\u4e91\u539f\u751f\u7f51\u7edc\u7f16\u6392\u7cfb\u7edf\uff0c\u5c06 SDN \u7684\u80fd\u529b\u548c\u4e91\u539f\u751f\u7ed3\u5408\uff0c \u63d0\u4f9b\u4e30\u5bcc\u7684\u529f\u80fd\uff0c\u6781\u81f4\u7684\u6027\u80fd\u4ee5\u53ca\u826f\u597d\u7684\u53ef\u8fd0\u7ef4\u6027\u3002</p> <p>\u4e30\u5bcc\u7684\u529f\u80fd\uff1a</p> <p>\u5982\u679c\u4f60\u6000\u5ff5 SDN \u9886\u57df\u4e30\u5bcc\u7684\u7f51\u7edc\u80fd\u529b\u5374\u5728\u4e91\u539f\u751f\u9886\u57df\u82e6\u82e6\u8ffd\u5bfb\u800c\u4e0d\u5f97\uff0c\u90a3\u4e48 Kube-OVN \u5c06\u662f\u4f60\u7684\u6700\u4f73\u9009\u62e9\u3002</p> <p>\u501f\u52a9 OVS/OVN \u5728 SDN \u9886\u57df\u6210\u719f\u7684\u80fd\u529b\uff0cKube-OVN \u5c06\u7f51\u7edc\u865a\u62df\u5316\u7684\u4e30\u5bcc\u529f\u80fd\u5e26\u5165\u4e91\u539f\u751f\u9886\u57df\u3002\u76ee\u524d\u5df2\u652f\u6301\u5b50\u7f51\u7ba1\u7406\uff0c \u9759\u6001 IP \u5206\u914d\uff0c\u5206\u5e03\u5f0f/\u96c6\u4e2d\u5f0f\u7f51\u5173\uff0cUnderlay/Overlay \u6df7\u5408\u7f51\u7edc\uff0c VPC \u591a\u79df\u6237\u7f51\u7edc\uff0c\u8de8\u96c6\u7fa4\u4e92\u8054\u7f51\u7edc\uff0cQoS \u7ba1\u7406\uff0c \u591a\u7f51\u5361\u7ba1\u7406\uff0cACL \u7f51\u7edc\u63a7\u5236\uff0c\u6d41\u91cf\u955c\u50cf\uff0cARM \u652f\u6301\uff0c Windows \u652f\u6301\u7b49\u8bf8\u591a\u529f\u80fd\u3002</p> <p>\u6781\u81f4\u7684\u6027\u80fd\uff1a</p> <p>\u5982\u679c\u4f60\u62c5\u5fc3\u5bb9\u5668\u7f51\u7edc\u4f1a\u5e26\u6765\u989d\u5916\u7684\u6027\u80fd\u635f\u8017\uff0c\u90a3\u4e48\u6765\u770b\u4e00\u4e0b Kube-OVN \u662f\u5982\u4f55\u6781\u81f4\u7684\u4f18\u5316\u6027\u80fd\u3002</p> <p>\u5728\u6570\u636e\u5e73\u9762\uff0c\u901a\u8fc7\u4e00\u7cfb\u5217\u5bf9\u6d41\u8868\u548c\u5185\u6838\u7684\u7cbe\u5fc3\u4f18\u5316\uff0c\u5e76\u501f\u52a9 eBPF\u3001DPDK\u3001\u667a\u80fd\u7f51\u5361\u5378\u8f7d\u7b49\u65b0\u5174\u6280\u672f\uff0c Kube-OVN \u53ef\u4ee5\u5728\u5ef6\u8fdf\u548c\u541e\u5410\u91cf\u7b49\u65b9\u9762\u7684\u6307\u6807\u8fbe\u5230\u8fd1\u4f3c\u6216\u8d85\u51fa\u5bbf\u4e3b\u673a\u7f51\u7edc\u6027\u80fd\u7684\u6c34\u5e73\u3002\u5728\u63a7\u5236\u5e73\u9762\uff0c\u901a\u8fc7\u5bf9 OVN \u4e0a\u6e38\u6d41\u8868\u7684\u88c1\u526a\uff0c \u5404\u79cd\u7f13\u5b58\u6280\u672f\u7684\u4f7f\u7528\u548c\u8c03\u4f18\uff0cKube-OVN \u53ef\u4ee5\u652f\u6301\u5927\u89c4\u6a21\u4e0a\u5343\u8282\u70b9\u548c\u4e0a\u4e07 Pod \u7684\u96c6\u7fa4\u3002</p> <p>\u6b64\u5916 Kube-OVN \u8fd8\u5728\u4e0d\u65ad\u4f18\u5316 CPU \u548c\u5185\u5b58\u7b49\u8d44\u6e90\u7684\u4f7f\u7528\u91cf\uff0c\u4ee5\u9002\u5e94\u8fb9\u7f18\u7b49\u8d44\u6e90\u6709\u9650\u573a\u666f\u3002</p> <p>\u826f\u597d\u7684\u53ef\u8fd0\u7ef4\u6027\uff1a</p> <p>\u5982\u679c\u4f60\u5bf9\u5bb9\u5668\u7f51\u7edc\u7684\u8fd0\u7ef4\u5fc3\u5b58\u5fe7\u8651\uff0cKube-OVN \u5185\u7f6e\u4e86\u5927\u91cf\u7684\u5de5\u5177\u6765\u5e2e\u52a9\u4f60\u7b80\u5316\u8fd0\u7ef4\u64cd\u4f5c\u3002</p> <p>Kube-OVN \u63d0\u4f9b\u4e86\u4e00\u952e\u5b89\u88c5\u811a\u672c\uff0c\u5e2e\u52a9\u7528\u6237\u8fc5\u901f\u642d\u5efa\u751f\u4ea7\u5c31\u7eea\u7684\u5bb9\u5668\u7f51\u7edc\u3002\u540c\u65f6\u5185\u7f6e\u7684\u4e30\u5bcc\u7684\u76d1\u63a7\u6307\u6807\u548c Grafana \u9762\u677f\uff0c \u53ef\u5e2e\u52a9\u7528\u6237\u5efa\u7acb\u5b8c\u5584\u7684\u76d1\u63a7\u4f53\u7cfb\u3002\u5f3a\u5927\u7684\u547d\u4ee4\u884c\u5de5\u5177\u53ef\u4ee5\u7b80\u5316\u7528\u6237\u7684\u65e5\u5e38\u8fd0\u7ef4\u64cd\u4f5c\u3002\u901a\u8fc7\u548c Cilium \u7ed3\u5408\uff0c\u5229\u7528 eBPF \u80fd\u529b\u7528\u6237\u53ef\u4ee5 \u589e\u5f3a\u5bf9\u7f51\u7edc\u7684\u53ef\u89c2\u6d4b\u6027\u3002 \u6b64\u5916\u6d41\u91cf\u955c\u50cf\u7684\u80fd\u529b\u53ef\u4ee5\u65b9\u4fbf\u7528\u6237\u81ea\u5b9a\u4e49\u6d41\u91cf\u76d1\u63a7\uff0c\u5e76\u548c\u4f20\u7edf\u7684 NPM \u7cfb\u7edf\u5bf9\u63a5\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"contact/","title":"\u8054\u7cfb\u65b9\u5f0f","text":"<p>\u5173\u6ce8\u516c\u4f17\u53f7\u83b7\u5f97\u66f4\u591a\u6700\u65b0\u4fe1\u606f\uff0c\u8bf7\u626b\u63cf\u4e0b\u65b9\u4e8c\u7ef4\u7801:</p> <p></p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/accelerate-intra-node-tcp-with-ebpf/","title":"\u4f7f\u7528 eBPF \u52a0\u901f\u8282\u70b9\u5185 TCP \u901a\u4fe1","text":"<p>\u5728\u4e00\u4e9b\u8fb9\u7f18\u548c 5G \u7684\u573a\u666f\u4e0b\uff0c\u540c\u8282\u70b9\u5185\u7684 Pod \u4e4b\u95f4\u4f1a\u8fdb\u884c\u5927\u91cf\u7684 TCP \u901a\u4fe1\uff0c\u901a\u8fc7\u4f7f\u7528 Intel \u5f00\u6e90\u7684 istio-tcpip-bypass \u9879\u76ee\uff0cPod \u53ef\u4ee5\u501f\u52a9 eBPF \u7684\u80fd\u529b\u7ed5\u8fc7\u4e3b\u673a\u7684 TCP/IP \u534f\u8bae\u6808\uff0c\u76f4\u63a5\u8fdb\u884c socket \u901a\u4fe1\uff0c\u4ece\u800c\u5927\u5e45\u964d\u4f4e\u5ef6\u8fdf\u5e76\u63d0\u5347\u541e\u5410\u91cf\u3002</p>"},{"location":"advance/accelerate-intra-node-tcp-with-ebpf/#_1","title":"\u57fa\u672c\u539f\u7406","text":"<p>\u5728\u5f53\u524d\u7684\u5b9e\u73b0\u4e0b\uff0c\u540c\u4e3b\u673a\u7684\u4e24\u4e2a Pod \u8fdb\u884c TCP \u8fdb\u884c\u901a\u4fe1\u9700\u8981\u7ecf\u8fc7\u5927\u91cf\u7684\u7f51\u7edc\u6808\uff0c\u5305\u62ec TCP/IP, netfilter\uff0cOVS \u7b49\u5982\u4e0b\u56fe\u6240\u793a\uff1a</p> <p></p> <p>istio-tcpip-bypass \u63d2\u4ef6\u53ef\u4ee5\u81ea\u52a8\u5206\u6790\u5e76\u8bc6\u522b\u51fa\u540c\u4e3b\u673a\u5185\u7684 TCP \u901a\u4fe1\uff0c\u5e76\u7ed5\u8fc7\u590d\u6742\u7684\u5185\u6838\u6808\u4ece\u800c\u53ef\u4ee5\u76f4\u63a5\u8fdb\u884c socket \u95f4\u7684\u6570\u636e\u4f20\u8f93\uff0c \u6765\u964d\u4f4e\u7f51\u7edc\u6808\u5904\u7406\u5f00\u9500\uff0c\u5982\u4e0b\u56fe\u6240\u793a\uff1a</p> <p></p> <p>\u7531\u4e8e\u8be5\u7ec4\u4ef6\u53ef\u4ee5\u81ea\u52a8\u8bc6\u522b\u540c\u4e3b\u673a\u5185\u7684 TCP \u901a\u4fe1\uff0c\u5e76\u8fdb\u884c\u4f18\u5316\u3002\u5728\u57fa\u4e8e\u4ee3\u7406\u6a21\u5f0f\u7684 Service Mesh \u73af\u5883\u4e0b\uff0c\u8be5\u7ec4\u4ef6\u4e5f\u53ef\u4ee5\u589e\u5f3a Service Mesh \u7684\u6027\u80fd\u8868\u73b0\u3002</p> <p>\u66f4\u591a\u6280\u672f\u5b9e\u73b0\u7ec6\u8282\u53ef\u4ee5\u53c2\u8003 Tanzu Service Mesh Acceleration using eBPF\u3002</p>"},{"location":"advance/accelerate-intra-node-tcp-with-ebpf/#_2","title":"\u73af\u5883\u51c6\u5907","text":"<p>eBPF \u5bf9\u5185\u6838\u7248\u672c\u6709\u4e00\u5b9a\u8981\u6c42\uff0c\u63a8\u8350\u4f7f\u7528 Ubuntu 20.04 \u548c Linux 5.4.0-74-generic \u7248\u672c\u5185\u6838\u8fdb\u884c\u5b9e\u9a8c\u3002</p>"},{"location":"advance/accelerate-intra-node-tcp-with-ebpf/#_3","title":"\u5b9e\u9a8c\u6b65\u9aa4","text":"<p>\u5728\u540c\u4e00\u4e2a\u8282\u70b9\u4e0a\u90e8\u7f72\u4e24\u4e2a\u6027\u80fd\u6d4b\u8bd5 Pod\uff0c\u82e5\u96c6\u7fa4\u5185\u5b58\u5728\u591a\u53f0\u673a\u5668\u9700\u8981\u6307\u5b9a <code>nodeSelector</code>\uff1a</p> <pre><code># kubectl create deployment perf --image=kubeovn/perf:dev --replicas=2\ndeployment.apps/perf created\n# kubectl get pod -o wide\nNAME                    READY   STATUS    RESTARTS   AGE   IP           NODE     NOMINATED NODE   READINESS GATES\nperf-7697bc6ddf-b2cpv   1/1     Running   0          28s   100.64.0.3   sealos   &lt;none&gt;           &lt;none&gt;\nperf-7697bc6ddf-p2xpt   1/1     Running   0          28s   100.64.0.2   sealos   &lt;none&gt;           &lt;none&gt;\n</code></pre> <p>\u8fdb\u5165\u5176\u4e2d\u4e00\u4e2a Pod \u5f00\u542f qperf server\uff0c\u5728\u53e6\u4e00\u4e2a Pod \u4e2d\u542f\u52a8 qperf client \u8fdb\u884c\u6027\u80fd\u6d4b\u8bd5\uff1a</p> <pre><code># kubectl exec -it perf-7697bc6ddf-b2cpv sh\n/ # qperf\n\n# kubectl exec -it perf-7697bc6ddf-p2xpt sh\n/ # qperf -t 60 100.64.0.3 -ub -oo msg_size:1:16K:*4 -vu tcp_lat tcp_bw\n</code></pre> <p>\u90e8\u7f72 istio-tcpip-bypass \u63d2\u4ef6\uff1a</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/intel/istio-tcpip-bypass/main/bypass-tcpip-daemonset.yaml\n</code></pre> <p>\u518d\u6b21\u8fdb\u5165 perf client \u5bb9\u5668\u8fdb\u884c\u6027\u80fd\u6d4b\u8bd5\uff1a</p> <pre><code># kubectl exec -it perf-7697bc6ddf-p2xpt sh\n/ # qperf -t 60 100.64.0.3 -ub -oo msg_size:1:16K:*4 -vu tcp_lat tcp_bw\n</code></pre>"},{"location":"advance/accelerate-intra-node-tcp-with-ebpf/#_4","title":"\u6d4b\u8bd5\u7ed3\u679c","text":"<p>\u6839\u636e\u6d4b\u8bd5\u7ed3\u679c TCP \u5ef6\u8fdf\u5728\u4e0d\u540c\u6570\u636e\u5305\u5927\u5c0f\u7684\u60c5\u51b5\u4e0b\u4f1a\u6709 40% ~ 60% \u7684\u5ef6\u8fdf\u4e0b\u964d\uff0c\u5728\u6570\u636e\u5305\u5927\u4e8e 1024 \u5b57\u8282\u65f6\u541e\u5410\u91cf\u4f1a\u6709 40% ~ 80% \u63d0\u5347\u3002</p> Packet Size (byte) eBPF tcp_lat (us) Default tcp_lat (us) eBPF tcp_bw (Mb/s) Default tcp_bw(Mb/s) 1 20.2 44.5 1.36 4.27 4 20.2 48.7 5.48 16.7 16 19.6 41.6 21.7 63.5 64 18.8 41.3 96.8 201 256 19.2 36 395 539 1024 18.3 42.4 1360 846 4096 16.5 62.6 4460 2430 16384 20.2 58.8 9600 6900 <p>\u5728\u6d4b\u8bd5\u7684\u786c\u4ef6\u73af\u5883\u4e0b\uff0c\u6570\u636e\u5305\u5c0f\u4e8e 512 \u5b57\u8282\u65f6\uff0c\u4f7f\u7528 eBPF \u4f18\u5316\u541e\u5410\u91cf\u6307\u6807\u4f1a\u4f4e\u4e8e\u9ed8\u8ba4\u914d\u7f6e\u4e0b\u7684\u541e\u5410\u91cf\u3002 \u8be5\u60c5\u51b5\u53ef\u80fd\u548c\u9ed8\u8ba4\u914d\u7f6e\u4e0b\u7f51\u5361\u5f00\u542f TCP \u805a\u5408\u4f18\u5316\u76f8\u5173\u3002\u5982\u679c\u5e94\u7528\u573a\u666f\u5bf9\u5c0f\u5305\u541e\u5410\u91cf\u654f\u611f\uff0c\u9700\u8981\u5728\u76f8\u5e94\u73af\u5883\u4e0b \u8fdb\u884c\u6d4b\u8bd5\u5224\u65ad\u662f\u5426\u5f00\u542f eBPF \u4f18\u5316\u3002\u6211\u4eec\u4e5f\u4f1a\u540e\u7eed\u5bf9 eBPF TCP \u5c0f\u5305\u573a\u666f\u7684\u541e\u5410\u91cf\u8fdb\u884c\u4f18\u5316\u3002</p>"},{"location":"advance/accelerate-intra-node-tcp-with-ebpf/#_5","title":"\u53c2\u8003\u8d44\u6599","text":"<ol> <li>istio-tcpip-bypass</li> <li>Deep Dive TCP/IP Bypass with eBPF in Service Mesh</li> <li>Tanzu Service Mesh Acceleration using eBPF</li> </ol> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/cilium-hubble-observe/","title":"Cilium \u7f51\u7edc\u6d41\u91cf\u89c2\u6d4b","text":"<p>Kube-OVN \u5f53\u524d\u5df2\u7ecf\u652f\u6301\u4e0e Cilium \u96c6\u6210\uff0c\u5177\u4f53\u64cd\u4f5c\u53ef\u4ee5\u53c2\u8003 Cilium \u96c6\u6210\u3002</p> <p>Cilium \u63d0\u4f9b\u4e86\u4e30\u5bcc\u7684\u7f51\u7edc\u6d41\u91cf\u89c2\u6d4b\u80fd\u529b\uff0c\u6d41\u91cf\u53ef\u89c2\u6d4b\u6027\u662f\u7531 Hubble \u63d0\u4f9b\u7684\u3002Hubble \u53ef\u4ee5\u89c2\u5bdf\u8282\u70b9\u3001\u96c6\u7fa4\u751a\u81f3\u591a\u96c6\u7fa4\u573a\u666f\u4e0b\u8de8\u96c6\u7fa4\u7684\u6d41\u91cf\u3002</p>"},{"location":"advance/cilium-hubble-observe/#hubble","title":"\u5b89\u88c5 Hubble","text":"<p>\u9ed8\u8ba4\u7684 Cilium \u96c6\u6210\u5b89\u88c5\u4e2d\uff0c\u5e76\u6ca1\u6709\u5b89\u88c5 Hubble \u76f8\u5173\u7ec4\u4ef6\uff0c\u56e0\u6b64\u8981\u652f\u6301\u6d41\u91cf\u89c2\u6d4b\uff0c\u9700\u8981\u5148\u5728\u73af\u5883\u4e0a\u8865\u5145\u5b89\u88c5 Hubble\u3002</p> <p>\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u4f7f\u7528 helm \u5b89\u88c5 Hubble\uff1a</p> <pre><code>helm upgrade cilium cilium/cilium --version 1.11.6 \\\n--namespace kube-system \\\n--reuse-values \\\n--set hubble.relay.enabled=true \\\n--set hubble.ui.enabled=true\n</code></pre> <p>\u8865\u5145\u5b89\u88c5 Hubble \u4e4b\u540e\uff0c\u6267\u884c <code>cilium status</code> \u67e5\u770b\u7ec4\u4ef6\u72b6\u6001\uff0c\u786e\u8ba4 Hubble \u5b89\u88c5\u6210\u529f\u3002</p> <pre><code># cilium status\n/\u00af\u00af\\\n/\u00af\u00af\\__/\u00af\u00af\\    Cilium:         OK\n \\__/\u00af\u00af\\__/    Operator:       OK\n /\u00af\u00af\\__/\u00af\u00af\\    Hubble:         OK\n \\__/\u00af\u00af\\__/    ClusterMesh:    disabled\n    \\__/\n\nDeployment        hubble-relay       Desired: 1, Ready: 1/1, Available: 1/1\nDeployment        cilium-operator    Desired: 2, Ready: 2/2, Available: 2/2\nDaemonSet         cilium             Desired: 2, Ready: 2/2, Available: 2/2\nDeployment        hubble-ui          Desired: 1, Ready: 1/1, Available: 1/1\nContainers:       cilium             Running: 2\nhubble-ui          Running: 1\nhubble-relay       Running: 1\ncilium-operator    Running: 2\nCluster Pods:     16/17 managed by Cilium\nImage versions    hubble-relay       quay.io/cilium/hubble-relay:v1.11.6@sha256:fd9034a2d04d5b973f1e8ed44f230ea195b89c37955ff32e34e5aa68f3ed675a: 1\ncilium-operator    quay.io/cilium/operator-generic:v1.11.6@sha256:9f6063c7bcaede801a39315ec7c166309f6a6783e98665f6693939cf1701bc17: 2\ncilium             quay.io/cilium/cilium:v1.11.6@sha256:f7f93c26739b6641a3fa3d76b1e1605b15989f25d06625260099e01c8243f54c: 2\nhubble-ui          quay.io/cilium/hubble-ui:v0.9.0@sha256:0ef04e9a29212925da6bdfd0ba5b581765e41a01f1cc30563cef9b30b457fea0: 1\nhubble-ui          quay.io/cilium/hubble-ui-backend:v0.9.0@sha256:000df6b76719f607a9edefb9af94dfd1811a6f1b6a8a9c537cba90bf12df474b: 1\napple@bogon cilium %\n</code></pre> <p>\u5b89\u88c5 Hubble \u7ec4\u4ef6\u4e4b\u540e\uff0c\u9700\u8981\u5b89\u88c5\u547d\u4ee4\u884c\uff0c\u7528\u4e8e\u5728\u73af\u5883\u4e0a\u67e5\u770b\u6d41\u91cf\u4fe1\u606f\u3002 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u5b89\u88c5 Hubble CLI :</p> <pre><code>curl -L --fail --remote-name-all https://github.com/cilium/hubble/releases/download/v0.10.0/hubble-linux-amd64.tar.gz\nsudo tar xzvfC hubble-linux-amd64.tar.gz /usr/local/bin\n</code></pre>"},{"location":"advance/cilium-hubble-observe/#_1","title":"\u90e8\u7f72\u6d4b\u8bd5\u4e1a\u52a1","text":"<p>Cilium \u5b98\u65b9\u63d0\u4f9b\u4e86\u4e00\u4e2a\u6d41\u91cf\u6d4b\u8bd5\u7684\u90e8\u7f72\u65b9\u6848\uff0c\u53ef\u4ee5\u76f4\u63a5\u4f7f\u7528\u5b98\u65b9\u90e8\u7f72\u7684\u4e1a\u52a1\u8fdb\u884c\u6d4b\u8bd5\u3002</p> <p>\u6267\u884c\u547d\u4ee4 <code>cilium connectivity test</code>\uff0cCilium \u4f1a\u81ea\u52a8\u521b\u5efa <code>cilium-test</code> \u7684 Namespace\uff0c\u540c\u65f6\u5728 cilium-test \u4e0b\u90e8\u7f72\u6d4b\u8bd5\u4e1a\u52a1\u3002</p> <p>\u6b63\u5e38\u90e8\u7f72\u5b8c\u540e\uff0c\u53ef\u4ee5\u67e5\u770b cilium-test namespace \u4e0b\u7684\u8d44\u6e90\u4fe1\u606f\uff0c\u53c2\u8003\u5982\u4e0b\uff1a</p> <pre><code># kubectl get all -n cilium-test\nNAME                                  READY   STATUS    RESTARTS   AGE\npod/client-7df6cfbf7b-z5t2j           1/1     Running   0          21s\npod/client2-547996d7d8-nvgxg          1/1     Running   0          21s\npod/echo-other-node-d79544ccf-hl4gg   2/2     Running   0          21s\npod/echo-same-node-5d466d5444-ml7tc   2/2     Running   0          21s\n\nNAME                      TYPE       CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE\nservice/echo-other-node   NodePort   10.109.58.126   &lt;none&gt;        8080:32269/TCP   21s\nservice/echo-same-node    NodePort   10.108.70.32    &lt;none&gt;        8080:32490/TCP   21s\n\nNAME                              READY   UP-TO-DATE   AVAILABLE   AGE\ndeployment.apps/client            1/1     1            1           21s\ndeployment.apps/client2           1/1     1            1           21s\ndeployment.apps/echo-other-node   1/1     1            1           21s\ndeployment.apps/echo-same-node    1/1     1            1           21s\n\nNAME                                        DESIRED   CURRENT   READY   AGE\nreplicaset.apps/client-7df6cfbf7b           1         1         1       21s\nreplicaset.apps/client2-547996d7d8          1         1         1       21s\nreplicaset.apps/echo-other-node-d79544ccf   1         1         1       21s\nreplicaset.apps/echo-same-node-5d466d5444   1         1         1       21s\n</code></pre>"},{"location":"advance/cilium-hubble-observe/#_2","title":"\u4f7f\u7528\u547d\u4ee4\u884c\u8fdb\u884c\u6d41\u91cf\u89c2\u6d4b","text":"<p>\u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u7f51\u7edc\u6d41\u91cf\u89c2\u6d4b\u4ec5\u63d0\u4f9b\u6bcf\u4e2a\u8282\u70b9 Cilium \u4ee3\u7406\u89c2\u5bdf\u5230\u7684\u6d41\u91cf\u3002 \u53ef\u4ee5\u5728 <code>kube-system namespace</code> \u4e0b\u7684 Cilium \u4ee3\u7406 pod \u4e2d\u6267\u884c <code>hubble observe</code> \u547d\u4ee4\uff0c\u67e5\u770b\u8be5\u8282\u70b9\u4e0a\u7684\u6d41\u91cf\u4fe1\u606f\u3002</p> <pre><code># kubectl get pod -n kube-system -o wide\nNAME                                             READY   STATUS    RESTARTS   AGE     IP           NODE                     NOMINATED NODE   READINESS GATES\ncilium-d6h56                                     1/1     Running   0          2d20h   172.18.0.2   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;\ncilium-operator-5887f78bbb-c7sb2                 1/1     Running   0          2d20h   172.18.0.2   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;\ncilium-operator-5887f78bbb-wj8gt                 1/1     Running   0          2d20h   172.18.0.3   kube-ovn-control-plane   &lt;none&gt;           &lt;none&gt;\ncilium-tq5xb                                     1/1     Running   0          2d20h   172.18.0.3   kube-ovn-control-plane   &lt;none&gt;           &lt;none&gt;\nkube-ovn-pinger-7lgk8                            1/1     Running   0          21h     10.16.0.19   kube-ovn-control-plane   &lt;none&gt;           &lt;none&gt;\nkube-ovn-pinger-msvcn                            1/1     Running   0          21h     10.16.0.18   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;\n\n# kubectl exec -it -n kube-system cilium-d6h56 -- bash\nroot@kube-ovn-worker:/home/cilium# hubble observe --from-namespace kube-system\nJul 29 03:24:25.551: kube-system/kube-ovn-pinger-msvcn:35576 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: ACK, PSH)\nJul 29 03:24:25.561: kube-system/kube-ovn-pinger-msvcn:35576 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: RST)\nJul 29 03:24:25.561: kube-system/kube-ovn-pinger-msvcn:35576 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: ACK, RST)\nJul 29 03:24:25.572: kube-system/kube-ovn-pinger-msvcn:35578 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: SYN)\nJul 29 03:24:25.572: kube-system/kube-ovn-pinger-msvcn:35578 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: ACK)\nJul 29 03:24:25.651: kube-system/kube-ovn-pinger-msvcn:35578 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: ACK, PSH)\nJul 29 03:24:25.661: kube-system/kube-ovn-pinger-msvcn:35578 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: RST)\nJul 29 03:24:25.661: kube-system/kube-ovn-pinger-msvcn:35578 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: ACK, RST)\nJul 29 03:24:25.761: kube-system/kube-ovn-pinger-msvcn:52004 -&gt; 172.18.0.3:6443 to-stack FORWARDED (TCP Flags: ACK, PSH)\nJul 29 03:24:25.779: kube-system/kube-ovn-pinger-msvcn -&gt; kube-system/kube-ovn-pinger-7lgk8 to-stack FORWARDED (ICMPv4 EchoRequest)\nJul 29 03:24:25.779: kube-system/kube-ovn-pinger-msvcn &lt;- kube-system/kube-ovn-pinger-7lgk8 to-endpoint FORWARDED (ICMPv4 EchoReply)\nJul 29 03:24:25.866: kube-system/hubble-ui-7596f7ff6f-7j6f2:55836 &lt;- kube-system/hubble-relay-959988db5-zc5vv:4245 to-stack FORWARDED (TCP Flags: ACK)\nJul 29 03:24:25.866: kube-system/hubble-ui-7596f7ff6f-7j6f2:55836 &lt;- kube-system/hubble-relay-959988db5-zc5vv:80 to-endpoint FORWARDED (TCP Flags: ACK)\nJul 29 03:24:25.866: kube-system/hubble-ui-7596f7ff6f-7j6f2:55836 -&gt; kube-system/hubble-relay-959988db5-zc5vv:4245 to-stack FORWARDED (TCP Flags: ACK)\nJul 29 03:24:25.866: kube-system/hubble-ui-7596f7ff6f-7j6f2:55836 -&gt; kube-system/hubble-relay-959988db5-zc5vv:4245 to-endpoint FORWARDED (TCP Flags: ACK)\nJul 29 03:24:25.975: kube-system/kube-ovn-pinger-7lgk8 -&gt; kube-system/kube-ovn-pinger-msvcn to-endpoint FORWARDED (ICMPv4 EchoRequest)\nJul 29 03:24:25.975: kube-system/kube-ovn-pinger-7lgk8 &lt;- kube-system/kube-ovn-pinger-msvcn to-stack FORWARDED (ICMPv4 EchoReply)\nJul 29 03:24:25.979: kube-system/kube-ovn-pinger-msvcn -&gt; 172.18.0.3 to-stack FORWARDED (ICMPv4 EchoRequest)\nJul 29 03:24:26.037: kube-system/coredns-6d4b75cb6d-lbgjg:36430 -&gt; 172.18.0.3:6443 to-stack FORWARDED (TCP Flags: ACK)\nJul 29 03:24:26.282: kube-system/kube-ovn-pinger-msvcn -&gt; 172.18.0.2 to-stack FORWARDED (ICMPv4 EchoRequest)\n</code></pre> <p>\u90e8\u7f72 Hubble Relay \u540e\uff0cHubble \u53ef\u4ee5\u63d0\u4f9b\u5b8c\u6574\u7684\u96c6\u7fa4\u8303\u56f4\u7684\u7f51\u7edc\u6d41\u91cf\u89c2\u6d4b\u3002</p>"},{"location":"advance/cilium-hubble-observe/#_3","title":"\u914d\u7f6e\u7aef\u53e3\u8f6c\u53d1","text":"<p>\u4e3a\u4e86\u80fd\u6b63\u5e38\u8bbf\u95ee Hubble API\uff0c\u9700\u8981\u521b\u5efa\u7aef\u53e3\u8f6c\u53d1\uff0c\u5c06\u672c\u5730\u8bf7\u6c42\u8f6c\u53d1\u5230 Hubble Service\u3002\u53ef\u4ee5\u6267\u884c <code>kubectl port-forward deployment/hubble-relay -n kube-system 4245:4245</code> \u547d\u4ee4\uff0c\u5728\u5f53\u524d\u7ec8\u7aef\u5f00\u542f\u7aef\u53e3\u8f6c\u53d1\u3002</p> <p>\u7aef\u53e3\u8f6c\u53d1\u914d\u7f6e\u53ef\u4ee5\u53c2\u8003 \u7aef\u53e3\u8f6c\u53d1\u3002</p> <p><code>kubectl port-forward</code> \u547d\u4ee4\u4e0d\u4f1a\u8fd4\u56de\uff0c\u9700\u8981\u6253\u5f00\u53e6\u4e00\u4e2a\u7ec8\u7aef\u6765\u7ee7\u7eed\u6d4b\u8bd5\u3002</p> <p>\u914d\u7f6e\u5b8c\u7aef\u53e3\u8f6c\u53d1\u4e4b\u540e\uff0c\u5728\u7ec8\u7aef\u6267\u884c <code>hubble status</code> \u547d\u4ee4\uff0c\u5982\u679c\u6709\u7c7b\u4f3c\u5982\u4e0b\u8f93\u51fa\uff0c\u5219\u7aef\u53e3\u8f6c\u53d1\u914d\u7f6e\u6b63\u786e\uff0c\u53ef\u4ee5\u4f7f\u7528\u547d\u4ee4\u884c\u8fdb\u884c\u6d41\u91cf\u89c2\u6d4b\u3002</p> <pre><code># hubble status\nHealthcheck (via localhost:4245): Ok\nCurrent/Max Flows: 8,190/8,190 (100.00%)\nFlows/s: 22.86\nConnected Nodes: 2/2\n</code></pre>"},{"location":"advance/cilium-hubble-observe/#_4","title":"\u547d\u4ee4\u884c\u89c2\u6d4b","text":"<p>\u5728\u7ec8\u7aef\u4e0a\u6267\u884c <code>hubble observe</code> \u547d\u4ee4\uff0c\u67e5\u770b\u96c6\u7fa4\u7684\u6d41\u91cf\u4fe1\u606f\u3002 \u89c2\u6d4b\u5230\u7684 cilium-test \u76f8\u5173\u7684\u6d4b\u8bd5\u6d41\u91cf\u53c2\u8003\u5982\u4e0b\uff1a </p> <p>\u9700\u8981\u6ce8\u610f\u7684\u662f\uff0c <code>hubble observe</code> \u547d\u4ee4\u7684\u663e\u793a\u7ed3\u679c\uff0c\u662f\u5f53\u524d\u547d\u4ee4\u884c\u6267\u884c\u65f6\u67e5\u8be2\u5230\u7684\u6d41\u91cf\u4fe1\u606f\u3002\u591a\u6b21\u6267\u884c\u547d\u4ee4\u884c\uff0c\u53ef\u4ee5\u67e5\u770b\u5230\u4e0d\u540c\u7684\u6d41\u91cf\u4fe1\u606f\u3002 \u66f4\u591a\u8be6\u7ec6\u7684\u89c2\u6d4b\u4fe1\u606f\uff0c\u53ef\u4ee5\u6267\u884c <code>hubble help observe</code> \u547d\u4ee4\u67e5\u770b Hubble CLI \u7684\u8be6\u7ec6\u4f7f\u7528\u65b9\u5f0f\u3002</p>"},{"location":"advance/cilium-hubble-observe/#ui","title":"\u4f7f\u7528 UI \u8fdb\u884c\u6d41\u91cf\u89c2\u6d4b","text":"<p>\u6267\u884c <code>cilium status</code> \u547d\u4ee4\uff0c\u786e\u8ba4 Hubble UI \u5df2\u7ecf\u5b89\u88c5\u6210\u529f\u3002\u5728\u7b2c\u4e8c\u6b65\u7684 Hubble \u5b89\u88c5\u4e2d\uff0c\u5df2\u7ecf\u8865\u5145\u4e86 UI \u7684\u5b89\u88c5\u3002</p> <p>\u6267\u884c\u547d\u4ee4 <code>cilium hubble ui</code> \u53ef\u4ee5\u81ea\u52a8\u521b\u5efa\u7aef\u53e3\u8f6c\u53d1\uff0c\u5c06 <code>hubble-ui service</code> \u6620\u5c04\u5230\u672c\u5730\u7aef\u53e3\u3002 \u6b63\u5e38\u60c5\u51b5\u4e0b\uff0c\u6267\u884c\u5b8c\u547d\u4ee4\u540e\uff0c\u4f1a\u81ea\u52a8\u6253\u5f00\u672c\u5730\u7684\u6d4f\u89c8\u5668\uff0c\u8df3\u8f6c\u5230 Hubble UI \u754c\u9762\u3002\u5982\u679c\u6ca1\u6709\u81ea\u52a8\u8df3\u8f6c\uff0c\u5728\u6d4f\u89c8\u5668\u4e2d\u8f93\u5165 <code>http://localhost:12000</code> \u6253\u5f00 UI \u89c2\u5bdf\u754c\u9762\u3002</p> <p>\u5728\u754c\u9762\u5de6\u4e0a\u89d2\uff0c\u9009\u62e9 <code>cilium-test</code> namespace\uff0c\u67e5\u770b Cilium \u63d0\u4f9b\u7684\u6d4b\u8bd5\u6d41\u91cf\u4fe1\u606f\u3002 </p>"},{"location":"advance/cilium-hubble-observe/#hubble_1","title":"Hubble \u6d41\u91cf\u76d1\u63a7","text":"<p>Hubble \u7ec4\u4ef6\u63d0\u4f9b\u4e86\u96c6\u7fa4\u4e2d Pod \u7f51\u7edc\u884c\u4e3a\u7684\u76d1\u63a7\uff0c\u4e3a\u4e86\u652f\u6301\u67e5\u770b Hubble \u63d0\u4f9b\u7684\u76d1\u63a7\u6570\u636e\uff0c\u9700\u8981\u4f7f\u80fd\u76d1\u63a7\u7edf\u8ba1\u3002</p> <p>\u53c2\u8003\u4ee5\u4e0b\u547d\u4ee4\uff0c\u8865\u5145 <code>hubble.metrics.enabled</code> \u914d\u7f6e\u9879:</p> <pre><code>helm upgrade cilium cilium/cilium --version 1.11.6 \\\n--namespace kube-system \\\n--reuse-values \\\n--set hubble.relay.enabled=true \\\n--set hubble.ui.enabled=true \\\n--set hubble.metrics.enabled=\"{dns,drop,tcp,flow,icmp,http}\"\n</code></pre> <p>\u90e8\u7f72\u4e4b\u540e\uff0c\u4f1a\u5728 kube-system namespace \u751f\u6210\u540d\u79f0\u4e3a <code>hubble-metrics</code> \u7684\u670d\u52a1\u3002\u901a\u8fc7\u8bbf\u95ee Endpoints \u67e5\u8be2 Hubble \u63d0\u4f9b\u7684\u76d1\u63a7\u6307\u6807\uff0c\u53c2\u8003\u5982\u4e0b:</p> <pre><code># curl 172.18.0.2:9091/metrics\n# HELP hubble_drop_total Number of drops\n# TYPE hubble_drop_total counter\nhubble_drop_total{protocol=\"ICMPv6\",reason=\"Unsupported L3 protocol\"} 2\n# HELP hubble_flows_processed_total Total number of flows processed\n# TYPE hubble_flows_processed_total counter\nhubble_flows_processed_total{protocol=\"ICMPv4\",subtype=\"to-endpoint\",type=\"Trace\",verdict=\"FORWARDED\"} 335\nhubble_flows_processed_total{protocol=\"ICMPv4\",subtype=\"to-stack\",type=\"Trace\",verdict=\"FORWARDED\"} 335\nhubble_flows_processed_total{protocol=\"ICMPv6\",subtype=\"\",type=\"Drop\",verdict=\"DROPPED\"} 2\nhubble_flows_processed_total{protocol=\"TCP\",subtype=\"to-endpoint\",type=\"Trace\",verdict=\"FORWARDED\"} 8282\nhubble_flows_processed_total{protocol=\"TCP\",subtype=\"to-stack\",type=\"Trace\",verdict=\"FORWARDED\"} 6767\nhubble_flows_processed_total{protocol=\"UDP\",subtype=\"to-endpoint\",type=\"Trace\",verdict=\"FORWARDED\"} 1642\nhubble_flows_processed_total{protocol=\"UDP\",subtype=\"to-stack\",type=\"Trace\",verdict=\"FORWARDED\"} 1642\n# HELP hubble_icmp_total Number of ICMP messages\n# TYPE hubble_icmp_total counter\nhubble_icmp_total{family=\"IPv4\",type=\"EchoReply\"} 335\nhubble_icmp_total{family=\"IPv4\",type=\"EchoRequest\"} 335\nhubble_icmp_total{family=\"IPv4\",type=\"RouterSolicitation\"} 2\n# HELP hubble_tcp_flags_total TCP flag occurrences\n# TYPE hubble_tcp_flags_total counter\nhubble_tcp_flags_total{family=\"IPv4\",flag=\"FIN\"} 2043\nhubble_tcp_flags_total{family=\"IPv4\",flag=\"RST\"} 301\nhubble_tcp_flags_total{family=\"IPv4\",flag=\"SYN\"} 1169\nhubble_tcp_flags_total{family=\"IPv4\",flag=\"SYN-ACK\"} 1169\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/cilium-networkpolicy/","title":"Cilium NetworkPolicy \u652f\u6301","text":"<p>Kube-OVN \u5f53\u524d\u5df2\u7ecf\u652f\u6301\u4e0e Cilium \u96c6\u6210\uff0c\u5177\u4f53\u64cd\u4f5c\u53ef\u4ee5\u53c2\u8003 Cilium \u96c6\u6210\u3002</p> <p>\u5728\u96c6\u6210 Cilium \u4e4b\u540e\uff0c\u5c31\u53ef\u4ee5\u4f7f\u7528 Cilium \u4f18\u79c0\u7684\u7f51\u7edc\u7b56\u7565\u80fd\u529b\uff0c\u5b9e\u73b0\u5bf9\u6d41\u91cf\u8bbf\u95ee\u7684\u63a7\u5236\u3002\u4ee5\u4e0b\u6587\u6863\u63d0\u4f9b\u4e86\u5bf9 Cilium L3 \u548c L4 \u7f51\u7edc\u7b56\u7565\u80fd\u529b\u7684\u96c6\u6210\u9a8c\u8bc1\u3002</p>"},{"location":"advance/cilium-networkpolicy/#_1","title":"\u9a8c\u8bc1\u6b65\u9aa4","text":""},{"location":"advance/cilium-networkpolicy/#pod","title":"\u521b\u5efa\u6d4b\u8bd5 Pod","text":"<p>\u521b\u5efa namespace <code>test</code>\u3002\u53c2\u8003\u4ee5\u4e0b yaml\uff0c\u5728 test namespace \u4e2d\u521b\u5efa\u6307\u5b9a label <code>app=test</code> \u7684 Pod\uff0c\u4f5c\u4e3a\u6d4b\u8bd5\u8bbf\u95ee\u7684\u76ee\u7684 Pod\u3002</p> <pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nlabels:\napp: test\nname: test\nnamespace: test\nspec:\nreplicas: 1\nselector:\nmatchLabels:\napp: test\nstrategy:\nrollingUpdate:\nmaxSurge: 25%\nmaxUnavailable: 25%\ntype: RollingUpdate\ntemplate:\nmetadata:\nlabels:\napp: test\nspec:\ncontainers:\n- image: docker.io/library/nginx:alpine\nimagePullPolicy: IfNotPresent\nname: nginx\n</code></pre> <p>\u540c\u6837\u53c2\u8003\u4ee5\u4e0b yaml\uff0c\u5728 default namespace \u4e0b\u521b\u5efa\u6307\u5b9a label <code>app=dynamic</code> \u7684 Pod \u4e3a\u53d1\u8d77\u8bbf\u95ee\u6d4b\u8bd5\u7684 Pod\u3002</p> <pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nlabels:\napp: dynamic\nname: dynamic\nnamespace: default\nspec:\nreplicas: 2\nselector:\nmatchLabels:\napp: dynamic\nstrategy:\nrollingUpdate:\nmaxSurge: 25%\nmaxUnavailable: 25%\ntype: RollingUpdate\ntemplate:\nmetadata:\ncreationTimestamp: null\nlabels:\napp: dynamic\nspec:\ncontainers:\n- image: docker.io/library/nginx:alpine\nimagePullPolicy: IfNotPresent\nname: nginx\n</code></pre> <p>\u67e5\u770b\u6d4b\u8bd5 Pod \u4ee5\u53ca Label \u4fe1\u606f:</p> <pre><code># kubectl get pod -o wide --show-labels\nNAME                         READY   STATUS    RESTARTS   AGE   IP           NODE                     NOMINATED NODE   READINESS GATES   LABELS\ndynamic-7d8d7874f5-9v5c4     1/1     Running   0          28h   10.16.0.35   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;            app=dynamic,pod-template-hash=7d8d7874f5\ndynamic-7d8d7874f5-s8z2n     1/1     Running   0          28h   10.16.0.36   kube-ovn-control-plane   &lt;none&gt;           &lt;none&gt;            app=dynamic,pod-template-hash=7d8d7874f5\n# kubectl get pod -o wide -n test --show-labels\nNAME                           READY   STATUS    RESTARTS   AGE     IP           NODE                     NOMINATED NODE   READINESS GATES   LABELS\ndynamic-7d8d7874f5-6dsg6       1/1     Running   0          7h20m   10.16.0.2    kube-ovn-control-plane   &lt;none&gt;           &lt;none&gt;            app=dynamic,pod-template-hash=7d8d7874f5\ndynamic-7d8d7874f5-tjgtp       1/1     Running   0          7h46m   10.16.0.42   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;            app=dynamic,pod-template-hash=7d8d7874f5\nlabel-test1-77b6764857-swq4k   1/1     Running   0          3h43m   10.16.0.12   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;            app=test1,pod-template-hash=77b6764857\n\n// \u4ee5\u4e0b\u4e3a\u6d4b\u8bd5\u8bbf\u95ee\u76ee\u7684 Pod\ntest-54c98bc466-mft5s          1/1     Running   0          8h      10.16.0.41   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;            app=test,pod-template-hash=54c98bc466\n</code></pre>"},{"location":"advance/cilium-networkpolicy/#l3","title":"L3 \u7f51\u7edc\u7b56\u7565\u6d4b\u8bd5","text":"<p>\u53c2\u8003\u4ee5\u4e0b yaml\uff0c\u521b\u5efa <code>CiliumNetworkPolicy</code> \u8d44\u6e90:</p> <pre><code>apiVersion: \"cilium.io/v2\"\nkind: CiliumNetworkPolicy\nmetadata:\nname: \"l3-rule\"\nnamespace: test\nspec:\nendpointSelector:\nmatchLabels:\napp: test\ningress:\n- fromEndpoints:\n- matchLabels:\napp: dynamic\n</code></pre> <p>\u5728 default namespace \u4e0b\u7684\u6d4b\u8bd5 Pod \u4e2d\uff0c\u53d1\u8d77\u5bf9\u76ee\u7684 Pod \u7684\u8bbf\u95ee\uff0c\u7ed3\u679c\u8bbf\u95ee\u4e0d\u901a\u3002 \u4f46\u662f\u5728 test namespace \u4e0b\uff0c\u6d4b\u8bd5\u5230\u76ee\u7684 Pod \u7684\u8bbf\u95ee\uff0c\u6d4b\u8bd5\u6b63\u5e38\u3002</p> <p>default namespace \u4e0b\u6d4b\u8bd5\u7ed3\u679c:</p> <pre><code># kubectl exec -it dynamic-7d8d7874f5-9v5c4 -- bash\nbash-5.0# ping -c 3 10.16.0.41\nPING 10.16.0.41 (10.16.0.41): 56 data bytes\n\n--- 10.16.0.41 ping statistics ---\n3 packets transmitted, 0 packets received, 100% packet loss\n</code></pre> <p>test namepsace \u4e0b Pod \u7684\u6d4b\u8bd5\uff0c\u8bbf\u95ee\u6b63\u5e38:</p> <pre><code># kubectl exec -it -n test dynamic-7d8d7874f5-6dsg6 -- bash\nbash-5.0# ping -c 3 10.16.0.41\nPING 10.16.0.41 (10.16.0.41): 56 data bytes\n64 bytes from 10.16.0.41: seq=0 ttl=64 time=2.558 ms\n64 bytes from 10.16.0.41: seq=1 ttl=64 time=0.223 ms\n64 bytes from 10.16.0.41: seq=2 ttl=64 time=0.304 ms\n\n--- 10.16.0.41 ping statistics ---\n3 packets transmitted, 3 packets received, 0% packet loss\nround-trip min/avg/max = 0.223/1.028/2.558 ms\n</code></pre> <p>\u67e5\u770b Cilium \u5b98\u65b9\u6587\u6863\u89e3\u91ca\uff0c<code>CiliumNetworkPolicy</code> \u8d44\u6e90\u5c06\u9650\u5236\u63a7\u5236\u5728\u4e86 <code>Namespace</code> \u7ea7\u522b\u3002\u8be6\u7ec6\u4fe1\u606f\u53ef\u4ee5\u67e5\u770b Cilium \u9650\u5236\u3002</p> <p>\u5728\u6709\u7f51\u7edc\u7b56\u7565\u89c4\u5219\u5339\u914d\u7684\u60c5\u51b5\u4e0b\uff0c\u53ea\u6709\u540c\u4e00\u4e2a Namespace \u7684 Pod \uff0c\u624d\u53ef\u4ee5\u6309\u7167\u89c4\u5219\u8fdb\u884c\u8bbf\u95ee\uff0c\u9ed8\u8ba4\u62d2\u7edd\u5176\u4ed6 Namespace \u7684 Pod \u8fdb\u884c\u8bbf\u95ee\u3002</p> <p>\u5982\u679c\u60f3\u5b9e\u73b0\u8de8 Namespace \u7684\u8bbf\u95ee\uff0c\u9700\u8981\u5728\u89c4\u5219\u4e2d\u660e\u786e\u6307\u5b9a Namespace \u4fe1\u606f\u3002</p> <p>\u53c2\u8003\u6587\u6863\uff0c\u4fee\u6539 <code>CiliumNetworkPolicy</code> \u8d44\u6e90\uff0c\u589e\u52a0 namespace \u4fe1\u606f:</p> <pre><code>  ingress:\n- fromEndpoints:\n- matchLabels:\napp: dynamic\nk8s:io.kubernetes.pod.namespace: default    // \u63a7\u5236\u5176\u4ed6 Namespace \u4e0b\u7684 Pod \u8bbf\u95ee\n</code></pre> <p>\u67e5\u770b\u4fee\u6539\u540e\u7684 <code>CiliumNetworkPolicy</code> \u8d44\u6e90\u4fe1\u606f:</p> <pre><code># kubectl get cnp -n test  -o yaml l3-rule\napiVersion: cilium.io/v2\nkind: CiliumNetworkPolicy\nmetadata:\n  name: l3-rule\n  namespace: test\nspec:\n  endpointSelector:\n    matchLabels:\n      app: test\ningress:\n  - fromEndpoints:\n    - matchLabels:\n        app: dynamic\n    - matchLabels:\n        app: dynamic\n        k8s:io.kubernetes.pod.namespace: default\n</code></pre> <p>\u518d\u6b21\u6d4b\u8bd5 default namespace \u4e0b\u7684 Pod \u8bbf\u95ee\uff0c\u76ee\u7684 Pod \u8bbf\u95ee\u6b63\u5e38:</p> <pre><code># kubectl exec -it dynamic-7d8d7874f5-9v5c4 -n test -- bash\nbash-5.0# ping -c 3 10.16.0.41\nPING 10.16.0.41 (10.16.0.41): 56 data bytes\n64 bytes from 10.16.0.41: seq=0 ttl=64 time=2.383 ms\n64 bytes from 10.16.0.41: seq=1 ttl=64 time=0.115 ms\n64 bytes from 10.16.0.41: seq=2 ttl=64 time=0.142 ms\n\n--- 10.16.0.41 ping statistics ---\n3 packets transmitted, 3 packets received, 0% packet loss\nround-trip min/avg/max = 0.115/0.880/2.383 ms\n</code></pre> <p>\u4f7f\u7528\u6807\u51c6\u7684 Kubernetes \u7f51\u7edc\u7b56\u7565 networkpolicy\uff0c\u6d4b\u8bd5\u7ed3\u679c\u663e\u793a Cilium \u540c\u6837\u5c06\u8bbf\u95ee\u9650\u5236\u5728\u540c\u4e00\u4e2a Namespace \u5185\uff0c\u8de8 Namespace \u7684\u8bbf\u95ee\u662f\u7981\u6b62\u7684\u3002</p> <p>\u8fd9\u70b9\u4e0e Kube-OVN \u5b9e\u73b0\u662f\u4e0d\u540c\u7684\u3002Kube-OVN \u652f\u6301\u6807\u51c6\u7684 k8s \u7f51\u7edc\u7b56\u7565\uff0c\u9650\u5236\u4e86\u5177\u4f53 Namespace \u4e0b\u7684\u76ee\u7684 Pod\uff0c\u4f46\u662f\u5bf9\u6e90\u5730\u5740 Pod\uff0c\u662f\u6ca1\u6709 Namespace \u9650\u5236\u7684\uff0c\u4efb\u4f55 Namespace \u4e0b\u7b26\u5408\u9650\u5236\u89c4\u5219\u7684 Pod\uff0c\u90fd\u53ef\u4ee5\u5b9e\u73b0\u5bf9\u76ee\u7684 Pod \u7684\u8bbf\u95ee\u3002</p>"},{"location":"advance/cilium-networkpolicy/#l4","title":"L4 \u7f51\u7edc\u7b56\u7565\u6d4b\u8bd5","text":"<p>\u53c2\u8003\u4ee5\u4e0b yaml\uff0c\u521b\u5efa L4 \u5c42\u7684\u7f51\u7edc\u7b56\u7565\u8d44\u6e90:</p> <pre><code>apiVersion: \"cilium.io/v2\"\nkind: CiliumNetworkPolicy\nmetadata:\nname: \"l4-rule\"\nnamespace: test\nspec:\nendpointSelector:\nmatchLabels:\napp: test\ningress:\n- fromEndpoints:\n- matchLabels:\napp: dynamic\ntoPorts:\n- ports:\n- port: \"80\"\nprotocol: TCP\n</code></pre> <p>\u6d4b\u8bd5\u76f8\u540c Namespace \u4e0b\uff0c\u7b26\u5408\u7f51\u7edc\u7b56\u7565\u89c4\u5219\u7684 Pod \u7684\u8bbf\u95ee</p> <pre><code># kubectl exec -it -n test dynamic-7d8d7874f5-6dsg6 -- bash\nbash-5.0# ping -c 3 10.16.0.41\nPING 10.16.0.41 (10.16.0.41): 56 data bytes\n\n--- 10.16.0.41 ping statistics ---\n3 packets transmitted, 0 packets received, 100% packet loss\nbash-5.0#\nbash-5.0# curl 10.16.0.41:80\n&lt;html&gt;\n&lt;head&gt;\n        &lt;title&gt;Hello World!&lt;/title&gt;\n        &lt;link href='//fonts.googleapis.com/css?family=Open+Sans:400,700' rel='stylesheet' type='text/css'&gt;\n        &lt;style&gt;\n        body {\nbackground-color: white;\ntext-align: center;\npadding: 50px;\nfont-family: \"Open Sans\",\"Helvetica Neue\",Helvetica,Arial,sans-serif;\n}\n#logo {\nmargin-bottom: 40px;\n}\n&lt;/style&gt;\n&lt;/head&gt;\n&lt;body&gt;\n                &lt;h1&gt;Hello World!&lt;/h1&gt;\n                                &lt;h3&gt;Links found&lt;/h3&gt;\n        &lt;h3&gt;I am on  test-54c98bc466-mft5s&lt;/h3&gt;\n        &lt;h3&gt;Cookie                  =&lt;/h3&gt;\n                                        &lt;b&gt;KUBERNETES&lt;/b&gt; listening in 443 available at tcp://10.96.0.1:443&lt;br /&gt;\n                                                &lt;h3&gt;my name is hanhouchao!&lt;/h3&gt;\n                        &lt;h3&gt; RequestURI='/'&lt;/h3&gt;\n&lt;/body&gt;\n&lt;/html&gt;\n</code></pre> <p>\u76f8\u540c Namespace \u4e0b\uff0c\u4e0d\u7b26\u5408\u7f51\u7edc\u7b56\u7565\u89c4\u5219\u7684 Pod \u8bbf\u95ee\u6d4b\u8bd5</p> <pre><code># kubectl exec -it -n test label-test1-77b6764857-swq4k -- bash\nbash-5.0# ping -c 3 10.16.0.41\nPING 10.16.0.41 (10.16.0.41): 56 data bytes\n\n--- 10.16.0.41 ping statistics ---\n3 packets transmitted, 0 packets received, 100% packet loss\nbash-5.0#\nbash-5.0# curl -v 10.16.0.41:80 --connect-timeout 10\n*   Trying 10.16.0.41:80...\n* After 10000ms connect time, move on!\n* connect to 10.16.0.41 port 80 failed: Operation timed out\n* Connection timeout after 10001 ms\n* Closing connection 0\ncurl: (28) Connection timeout after 10001 ms\n</code></pre> <p>\u7f51\u7edc\u7b56\u7565\u751f\u6548\u540e\uff0c\u8de8 Namespace \u7684\u8bbf\u95ee\uff0c\u4f9d\u7136\u662f\u88ab\u7981\u6b62\u7684\uff0c\u8ddf L3 \u7f51\u7edc\u7b56\u7565\u6d4b\u8bd5\u7ed3\u679c\u4e00\u81f4\u3002</p> <p>\u5728 L4 \u7f51\u7edc\u89c4\u5219\u751f\u6548\u540e\uff0cping \u65e0\u6cd5\u4f7f\u7528\uff0c\u4f46\u662f\u7b26\u5408\u7b56\u7565\u89c4\u5219\u7684 TCP \u8bbf\u95ee\uff0c\u662f\u53ef\u4ee5\u6b63\u5e38\u6267\u884c\u7684\u3002</p> <p>\u5173\u4e8e ICMP \u7684\u9650\u5236\uff0c\u53ef\u4ee5\u53c2\u8003\u5b98\u65b9\u8bf4\u660e L4 \u9650\u5236\u8bf4\u660e\u3002</p>"},{"location":"advance/cilium-networkpolicy/#l7","title":"L7 \u7f51\u7edc\u7b56\u7565\u6d4b\u8bd5","text":"<p>chaining \u6a21\u5f0f\u4e0b\uff0cL7 \u7f51\u7edc\u7b56\u7565\u76ee\u524d\u662f\u5b58\u5728\u95ee\u9898\u7684\u3002\u5728 Cilium \u5b98\u65b9\u6587\u6863\u4e2d\uff0c\u5bf9\u8fd9\u79cd\u60c5\u51b5\u7ed9\u51fa\u4e86\u8bf4\u660e\uff0c\u53c2\u8003 Generic Veth Chaining\u3002</p> <p>\u8fd9\u4e2a\u95ee\u9898\u4f7f\u7528 issue 12454 \u8ddf\u8e2a\uff0c\u76ee\u524d\u8fd8\u6ca1\u6709\u89e3\u51b3\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/dhcp/","title":"DHCP \u8bbe\u7f6e","text":"<p>\u5728\u4f7f\u7528 SR-IOV \u6216 DPDK \u7c7b\u578b\u7f51\u7edc\u65f6\uff0cKubeVirt \u5185\u7f6e\u7684 DHCP \u65e0\u6cd5\u5728\u8be5\u7f51\u7edc\u6a21\u5f0f\u4e0b\u5de5\u4f5c\u3002Kube-OVN \u53ef\u4ee5\u5229\u7528 OVN \u7684 DHCP \u80fd\u529b\u5728\u5b50\u7f51\u7ea7\u522b\u8bbe\u7f6e  DHCP \u9009\u9879\uff0c\u4ece\u800c\u5e2e\u52a9\u8be5\u7f51\u7edc\u7c7b\u578b\u7684 KubeVirt \u865a\u673a\u6b63\u5e38\u4f7f\u7528 DHCP \u83b7\u5f97\u5206\u914d\u7684 IP \u5730\u5740\u3002Kube-OVN \u540c\u65f6\u652f\u6301 DHCPv4 \u548c DHCPv6\u3002</p> <p>\u5b50\u7f51 DHCP \u7684\u914d\u7f6e\u5982\u4e0b\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: sn-dual\nspec:\ncidrBlock: \"10.0.0.0/24,240e::a00/120\"\ndefault: false\ndisableGatewayCheck: true\ndisableInterConnection: false\nexcludeIps:\n- 10.0.0.1\n- 240e::a01\ngateway: 10.0.0.1,240e::a01\ngatewayNode: ''\ngatewayType: distributed\nnatOutgoing: false\nprivate: false\nprotocol: Dual\nprovider: ovn\nvpc: vpc-test\nenableDHCP: true\ndhcpV4Options: \"lease_time=3600,router=10.0.0.1,server_id=169.254.0.254,server_mac=00:00:00:2E:2F:B8\"\ndhcpV6Options: \"server_id=00:00:00:2E:2F:C5\"\nenableIPv6RA: true\nipv6RAConfigs: \"address_mode=dhcpv6_stateful,max_interval=30,min_interval=5,send_periodic=true\"\n</code></pre> <ul> <li><code>enableDHCP</code>: \u662f\u5426\u5f00\u542f\u5b50\u7f51\u7684 DHCP \u529f\u80fd\u3002</li> <li><code>dhcpV4Options</code>,<code>dhcpV6Options</code>: \u8be5\u5b57\u6bb5\u76f4\u63a5\u66b4\u9732 ovn-nb \u5185 DHCP \u76f8\u5173\u9009\u9879\uff0c\u8bf7\u53c2\u8003 DHCP Options\u3002 \u9ed8\u8ba4\u503c\u5206\u522b\u4e3a <code>\"lease_time=3600, router=$ipv4_gateway, server_id=169.254.0.254, server_mac=$random_mac\"</code> \u548c <code>server_id=$random_mac</code>\u3002</li> <li><code>enableIPv6RA</code>: \u662f\u5426\u5f00\u542f DHCPv6 \u7684\u8def\u7531\u5e7f\u64ad\u529f\u80fd\u3002</li> <li><code>ipv6RAConfigs</code>\uff1a\u8be5\u5b57\u6bb5\u76f4\u63a5\u66b4\u9732 ovn-nb \u5185 Logical_Router_Port \u76f8\u5173\u9009\u9879\uff0c\u8bf7\u53c2\u8003 Logical Router Port \u9ed8\u8ba4\u503c\u4e3a <code>address_mode=dhcpv6_stateful, max_interval=30, min_interval=5, send_periodic=true</code>\u3002</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/dpdk/","title":"DPDK \u652f\u6301","text":"<p>\u8be5\u6587\u6863\u4ecb\u7ecd Kube-OVN \u5982\u4f55\u548c OVS-DPDK \u7ed3\u5408\uff0c\u7ed9 KubeVirt \u7684\u865a\u673a\u63d0\u4f9b DPDK \u7c7b\u578b\u7684\u7f51\u7edc\u63a5\u53e3\u3002</p> <p>\u4e0a\u6e38\u7684 KubeVirt \u76ee\u524d\u8fd8\u672a\u652f\u6301 OVS-DPDK\uff0c\u7528\u6237\u9700\u8981\u81ea\u5df1\u901a\u8fc7\u76f8\u5173 patch Vhostuser implementation \u6784\u5efa KubeVirt \u6216 KVM Device Plugin \u6765\u4f7f\u7528 OVS-DPDK\u3002</p>"},{"location":"advance/dpdk/#_1","title":"\u524d\u63d0\u6761\u4ef6","text":"<ul> <li>\u8282\u70b9\u9700\u63d0\u4f9b\u4e13\u95e8\u7ed9 DPDK \u9a71\u52a8\u8fd0\u884c\u7684\u7f51\u5361\u3002</li> <li>\u8282\u70b9\u9700\u5f00\u542f Hugepages\u3002</li> </ul>"},{"location":"advance/dpdk/#dpdk_1","title":"\u7f51\u5361\u8bbe\u7f6e DPDK \u9a71\u52a8","text":"<p>\u8fd9\u91cc\u6211\u4eec\u4f7f\u7528 <code>driverctl</code> \u4e3a\u4f8b\u8fdb\u884c\u64cd\u4f5c\uff0c\u5177\u4f53\u53c2\u6570\u548c\u5176\u4ed6\u9a71\u52a8\u4f7f\u7528\u8bf7\u53c2\u8003 DPDK \u6587\u6863\u8fdb\u884c\u64cd\u4f5c\u3002</p> <pre><code>driverctl set-override 0000:00:0b.0 uio_pci_generic\n</code></pre>"},{"location":"advance/dpdk/#_2","title":"\u8282\u70b9\u914d\u7f6e","text":"<p>\u5bf9\u652f\u6301 OVS-DPDK \u7684\u8282\u70b9\u6253\u6807\u7b7e\uff0c\u4ee5\u4fbf Kube-OVN \u8fdb\u884c\u8bc6\u522b\u5904\u7406\uff1a</p> <pre><code>kubectl label nodes &lt;node&gt; ovn.kubernetes.io/ovs_dp_type=\"userspace\"\n</code></pre> <p>\u5728\u652f\u6301 OVS-DPDK \u8282\u70b9\u7684 <code>/opt/ovs-config</code> \u76ee\u5f55\u4e0b\u521b\u5efa\u914d\u7f6e\u6587\u4ef6 <code>ovs-dpdk-config</code>\uff1a</p> <pre><code>ENCAP_IP=192.168.122.193/24\nDPDK_DEV=0000:00:0b.0\n</code></pre> <ul> <li><code>ENCAP_IP</code>: \u96a7\u9053\u7aef\u70b9\u5730\u5740\u3002</li> <li><code>DPDK_DEV</code>: \u8bbe\u5907\u7684 PCI ID\u3002</li> </ul>"},{"location":"advance/dpdk/#kube-ovn","title":"\u5b89\u88c5 Kube-OVN","text":"<p>\u4e0b\u8f7d\u5b89\u88c5\u811a\u672c\uff1a</p> <pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/images/install.sh\n</code></pre> <p>\u542f\u7528 DPDK \u5b89\u88c5\u9009\u9879\u8fdb\u884c\u5b89\u88c5\uff1a</p> <pre><code>bash install.sh --with-hybrid-dpdk\n</code></pre>"},{"location":"advance/dpdk/#_3","title":"\u4f7f\u7528\u65b9\u5f0f","text":"<p>\u8fd9\u91cc\u6211\u4eec\u901a\u8fc7\u521b\u5efa\u4e00\u4e2a\u4f7f\u7528 vhostuser \u7c7b\u578b\u7f51\u5361\u7684\u865a\u673a\u6765\u9a8c\u8bc1 OVS-DPDK \u529f\u80fd\u3002</p> <p>\u5b89\u88c5 KVM Device Plugin \u6765\u521b\u5efa\u865a\u673a\uff0c\u66f4\u591a\u4f7f\u7528\u65b9\u5f0f\u8bf7\u53c2\u8003 KVM Device Plugin\u3002</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/kubevirt/kubernetes-device-plugins/master/manifests/kvm-ds.yml\n</code></pre> <p>\u521b\u5efa NetworkAttachmentDefinition\uff1a</p> <pre><code>apiVersion: k8s.cni.cncf.io/v1\nkind: NetworkAttachmentDefinition\nmetadata:\nname: ovn-dpdk\nnamespace: default\nspec:\nconfig: &gt;-\n{\n\"cniVersion\": \"0.3.0\", \n\"type\": \"kube-ovn\", \n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\", \n\"provider\": \"ovn-dpdk.default.ovn\",\n\"vhost_user_socket_volume_name\": \"vhostuser-sockets\",\n\"vhost_user_socket_name\": \"sock\"\n}\n</code></pre> <p>\u4f7f\u7528\u4e0b\u9762\u7684 Dockerfile \u521b\u5efa VM \u955c\u50cf\uff1a</p> <pre><code>FROM quay.io/kubevirt/virt-launcher:v0.46.1\n\n# wget http://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud.qcow2\nCOPY CentOS-7-x86_64-GenericCloud.qcow2 /var/lib/libvirt/images/CentOS-7-x86_64-GenericCloud.qcow2\n</code></pre> <p>\u521b\u5efa\u865a\u62df\u673a\uff1a</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: vm-config\ndata:\nstart.sh: |\nchmod u+w /etc/libvirt/qemu.conf\necho \"hugetlbfs_mount = \\\"/dev/hugepages\\\"\" &gt;&gt; /etc/libvirt/qemu.conf\nvirtlogd &amp;\nlibvirtd &amp;\n\nmkdir /var/lock\n\nsleep 5\n\nvirsh define /root/vm/vm.xml\nvirsh start vm\n\ntail -f /dev/null\nvm.xml: |\n&lt;domain type='kvm'&gt;\n&lt;name&gt;vm&lt;/name&gt;\n&lt;uuid&gt;4a9b3f53-fa2a-47f3-a757-dd87720d9d1d&lt;/uuid&gt;\n&lt;memory unit='KiB'&gt;2097152&lt;/memory&gt;\n&lt;currentMemory unit='KiB'&gt;2097152&lt;/currentMemory&gt;\n&lt;memoryBacking&gt;\n&lt;hugepages&gt;\n&lt;page size='2' unit='M' nodeset='0'/&gt;\n&lt;/hugepages&gt;\n&lt;/memoryBacking&gt;\n&lt;vcpu placement='static'&gt;2&lt;/vcpu&gt;\n&lt;cputune&gt;\n&lt;shares&gt;4096&lt;/shares&gt;\n&lt;vcpupin vcpu='0' cpuset='4'/&gt;\n&lt;vcpupin vcpu='1' cpuset='5'/&gt;\n&lt;emulatorpin cpuset='1,3'/&gt;\n&lt;/cputune&gt;\n&lt;os&gt;\n&lt;type arch='x86_64' machine='pc'&gt;hvm&lt;/type&gt;\n&lt;boot dev='hd'/&gt;\n&lt;/os&gt;\n&lt;features&gt;\n&lt;acpi/&gt;\n&lt;apic/&gt;\n&lt;/features&gt;\n&lt;cpu mode='host-model'&gt;\n&lt;model fallback='allow'/&gt;\n&lt;topology sockets='1' cores='2' threads='1'/&gt;\n&lt;numa&gt;\n&lt;cell id='0' cpus='0-1' memory='2097152' unit='KiB' memAccess='shared'/&gt;\n&lt;/numa&gt;\n&lt;/cpu&gt;\n&lt;on_reboot&gt;restart&lt;/on_reboot&gt;\n&lt;devices&gt;\n&lt;emulator&gt;/usr/libexec/qemu-kvm&lt;/emulator&gt;\n&lt;disk type='file' device='disk'&gt;\n&lt;driver name='qemu' type='qcow2' cache='none'/&gt;\n&lt;source file='/var/lib/libvirt/images/CentOS-7-x86_64-GenericCloud.qcow2'/&gt;\n&lt;target dev='vda' bus='virtio'/&gt;\n&lt;/disk&gt;\n\n&lt;interface type='vhostuser'&gt;\n&lt;mac address='00:00:00:0A:30:89'/&gt;\n&lt;source type='unix' path='/var/run/vm/sock' mode='server'/&gt;\n&lt;model type='virtio'/&gt;\n&lt;driver queues='2'&gt;\n&lt;host mrg_rxbuf='off'/&gt;\n&lt;/driver&gt;\n&lt;/interface&gt;\n&lt;serial type='pty'&gt;\n&lt;target type='isa-serial' port='0'&gt;\n&lt;model name='isa-serial'/&gt;\n&lt;/target&gt;\n&lt;/serial&gt;\n&lt;console type='pty'&gt;\n&lt;target type='serial' port='0'/&gt;\n&lt;/console&gt;\n&lt;channel type='unix'&gt;\n&lt;source mode='bind' path='/var/lib/libvirt/qemu/channel/target/domain-1-vm/org.qemu.guest_agent.0'/&gt;\n&lt;target type='virtio' name='org.qemu.guest_agent.0' state='connected'/&gt;\n&lt;alias name='channel0'/&gt;\n&lt;address type='virtio-serial' controller='0' bus='0' port='1'/&gt;\n&lt;/channel&gt;\n\n&lt;/devices&gt;\n&lt;/domain&gt;\n---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\nname: vm-deployment\nlabels:\napp: vm\nspec:\nreplicas: 1\nselector:\nmatchLabels:\napp: vm\ntemplate:\nmetadata:\nlabels:\napp: vm\nannotations:\nk8s.v1.cni.cncf.io/networks: default/ovn-dpdk\novn-dpdk.default.ovn.kubernetes.io/ip_address: 10.16.0.96\novn-dpdk.default.ovn.kubernetes.io/mac_address: 00:00:00:0A:30:89\nspec:\nnodeSelector:\novn.kubernetes.io/ovs_dp_type: userspace\nsecurityContext:\nrunAsUser: 0\nvolumes:\n- name: vhostuser-sockets\nemptyDir: {}\n- name: xml\nconfigMap:\nname: vm-config\n- name: hugepage\nemptyDir:\nmedium: HugePages-2Mi\n- name: libvirt-runtime\nemptyDir: {}\ncontainers:\n- name: vm\nimage: vm-vhostuser:latest\ncommand: [\"bash\", \"/root/vm/start.sh\"]\nsecurityContext:\ncapabilities:\nadd:\n- NET_BIND_SERVICE\n- SYS_NICE\n- NET_RAW\n- NET_ADMIN\nprivileged: false\nrunAsUser: 0\nresources:\nlimits:\ncpu: '2'\ndevices.kubevirt.io/kvm: '1'\nmemory: '8784969729'\nhugepages-2Mi: 2Gi\nrequests:\ncpu: 666m\ndevices.kubevirt.io/kvm: '1'\nephemeral-storage: 50M\nmemory: '4490002433'\nvolumeMounts:\n- name: vhostuser-sockets\nmountPath: /var/run/vm\n- name: xml\nmountPath: /root/vm/\n- mountPath: /dev/hugepages\nname: hugepage\n- name: libvirt-runtime\nmountPath: /var/run/libvirt\n</code></pre> <p>\u7b49\u5f85\u865a\u62df\u673a\u521b\u5efa\u6210\u529f\u540e\u8fdb\u5165 Pod \u8fdb\u884c\u865a\u673a\u914d\u7f6e\uff1a</p> <pre><code># virsh set-user-password vm root 12345\nPassword set successfully for root in vm\n\n# virsh console vm\nConnected to domain 'vm'\nEscape character is ^] (Ctrl + ])\n\nCentOS Linux 7 (Core)\nKernel 3.10.0-1127.el7.x86_64 on an x86_64\n\nlocalhost login: root\nPassword:\nLast login: Fri Feb 25 09:52:54 on ttyS0\n</code></pre> <p>\u63a5\u4e0b\u6765\u53ef\u4ee5\u767b\u5f55\u865a\u673a\u8fdb\u884c\u7f51\u7edc\u914d\u7f6e\u5e76\u6d4b\u8bd5\uff1a</p> <pre><code>ip link set eth0 mtu 1400\nip addr add 10.16.0.96/16 dev eth0\nip ro add default via 10.16.0.1\nping 114.114.114.114\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/external-gateway/","title":"\u5916\u90e8\u7f51\u5173\u8bbe\u7f6e","text":"<p>\u5728\u4e00\u4e9b\u573a\u666f\u4e0b\uff0c\u5bf9\u6240\u6709\u5bb9\u5668\u8bbf\u95ee\u5916\u90e8\u7684\u6d41\u91cf\u9700\u8981\u901a\u8fc7\u4e00\u4e2a\u5916\u90e8\u7684\u7f51\u5173\u8fdb\u884c\u7edf\u4e00\u7684\u7ba1\u7406\u548c\u5ba1\u8ba1\u3002 Kube-OVN \u53ef\u4ee5\u901a\u8fc7\u5728\u5b50\u7f51\u4e2d\u8fdb\u884c\u76f8\u5e94\u7684\u8def\u7531\u914d\u7f6e\uff0c\u5c06\u51fa\u7f51\u6d41\u91cf\u8f6c\u53d1\u81f3\u5bf9\u5e94\u7684\u5916\u90e8\u7f51\u5173\u3002</p>"},{"location":"advance/external-gateway/#_2","title":"\u4f7f\u7528\u65b9\u5f0f","text":"<pre><code>kind: Subnet\napiVersion: kubeovn.io/v1\nmetadata:\nname: external\nspec:\ncidrBlock: 172.31.0.0/16\ngatewayType: centralized\nnatOutgoing: false\nexternalEgressGateway: 192.168.0.1\npolicyRoutingTableID: 1000\npolicyRoutingPriority: 1500\n</code></pre> <ul> <li><code>natOutgoing</code>: \u9700\u8981\u8bbe\u7f6e\u4e3a <code>false</code>\u3002</li> <li><code>externalEgressGateway</code>\uff1a\u8bbe\u7f6e\u4e3a\u5916\u90e8\u7f51\u5173\u7684\u5730\u5740\uff0c\u9700\u8981\u548c\u7f51\u5173\u8282\u70b9\u5728\u540c\u4e00\u4e2a\u4e8c\u5c42\u53ef\u8fbe\u57df\u3002</li> <li><code>policyRoutingTableID</code>\uff1a\u4f7f\u7528\u7684\u672c\u5730\u7b56\u7565\u8def\u7531\u8868\u7684 TableID \u6bcf\u4e2a\u5b50\u7f51\u5747\u9700\u4e0d\u540c\u4ee5\u907f\u514d\u51b2\u7a81\u3002</li> <li><code>policyRoutingPriority</code>\uff1a\u8def\u7531\u4f18\u5148\u7ea7\uff0c\u4e3a\u907f\u514d\u540e\u7eed\u7528\u6237\u5b9a\u5236\u5316\u7684\u5176\u4ed6\u8def\u7531\u64cd\u4f5c\u51b2\u7a81\uff0c\u8fd9\u91cc\u53ef\u4ee5\u6307\u5b9a\u8def\u7531\u4f18\u5148\u7ea7\uff0c\u82e5\u65e0\u7279\u6b8a\u9700\u6c42\u586b\u5165\u4efb\u610f\u503c\u5373\u53ef\u3002</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/fastpath/","title":"\u624b\u52a8\u7f16\u8bd1 FastPath \u6a21\u5757","text":"<p>\u7ecf\u8fc7\u6570\u636e\u5e73\u9762\u7684\u6027\u80fd Profile\uff0c<code>Netfilter</code> \u5728\u5bb9\u5668\u5185\u548c\u5bbf\u4e3b\u673a\u4e0a\u7684\u76f8\u5173\u5904\u7406\u6d88\u8017\u4e86 20% \u5de6\u53f3\u7684 CPU \u8d44\u6e90\uff0cFastPath \u6a21\u5757\u53ef\u4ee5\u7ed5\u8fc7 <code>Netfilter</code> \u4ece\u800c \u964d\u4f4e CPU \u7684\u6d88\u8017\u548c\u5ef6\u8fdf\uff0c\u5e76\u63d0\u5347\u541e\u5410\u91cf\u3002\u672c\u6587\u6863\u5c06\u4ecb\u7ecd\u5982\u4f55\u624b\u52a8\u7f16\u8bd1 FastPath \u6a21\u5757\u3002</p>"},{"location":"advance/fastpath/#_1","title":"\u4e0b\u8f7d\u76f8\u5173\u5185\u6838\u6a21\u5757\u4ee3\u7801","text":"<pre><code>git clone --depth=1 https://github.com/kubeovn/kube-ovn.git\n</code></pre>"},{"location":"advance/fastpath/#_2","title":"\u5b89\u88c5\u4f9d\u8d56","text":"<p>\u8fd9\u91cc\u4ee5 CentOS \u4e3a\u4f8b\u4e0b\u8f7d\u76f8\u5173\u4f9d\u8d56\uff1a</p> <pre><code>yum install -y kernel-devel-$(uname -r) gcc elfutils-libelf-devel\n</code></pre>"},{"location":"advance/fastpath/#_3","title":"\u7f16\u8bd1\u76f8\u5173\u6a21\u5757","text":"<p>\u9488\u5bf9 3.x \u7684\u5185\u6838\uff1a</p> <pre><code>cd kube-ovn/fastpath\nmake all\n</code></pre> <p>\u9488\u5bf9 4.x \u7684\u5185\u6838\uff1a</p> <pre><code>cd kube-ovn/fastpath/4.18\ncp ../Makefile .\nmake all\n</code></pre>"},{"location":"advance/fastpath/#_4","title":"\u5b89\u88c5\u5185\u6838\u6a21\u5757","text":"<p>\u5c06 <code>kube_ovn_fastpath.ko</code> \u590d\u5236\u5230\u6bcf\u4e2a\u9700\u8981\u6027\u80fd\u4f18\u5316\u7684\u8282\u70b9 <code>/tmp</code> \u76ee\u5f55\u4e0b\uff0c<code>kube-ovn-cni</code> \u4f1a\u81ea\u52a8\u52a0\u8f7d\u8be5\u6a21\u5757\u3002</p> <p>\u4f7f\u7528 <code>dmesg</code> \u786e\u8ba4\u5b89\u88c5\u6210\u529f\uff1a</p> <pre><code># dmesg\n[619631.323788] init_module,kube_ovn_fastpath_local_out\n[619631.323798] init_module,kube_ovn_fastpath_post_routing\n[619631.323800] init_module,kube_ovn_fastpath_pre_routing\n[619631.323801] init_module,kube_ovn_fastpath_local_in\n</code></pre> <p>\u5982\u9700\u5378\u8f7d\u6a21\u5757\uff0c\u53ef\u5c06\u8be5\u6a21\u5757\u4ece <code>/tmp</code> \u76ee\u5f55\u4e0b\u79fb\u9664\uff0c<code>kube-ovn-cni</code> \u4f1a\u81ea\u52a8\u5378\u8f7d\u8be5\u6a21\u5757\u3002</p> <p>\u8be5\u6a21\u5757\u5728\u673a\u5668\u91cd\u542f\u540e\u4e0d\u4f1a\u81ea\u52a8\u52a0\u8f7d\uff0c\u5982\u9700\u81ea\u52a8\u52a0\u8f7d\u8bf7\u6839\u636e\u7cfb\u7edf\u5f04\u914d\u7f6e\u7f16\u5199\u76f8\u5e94\u81ea\u542f\u52a8\u811a\u672c\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/multi-nic/","title":"\u591a\u7f51\u5361\u7ba1\u7406","text":"<p>Kube-OVN \u53ef\u4ee5\u4e3a\u5176\u4ed6 CNI \u7f51\u7edc\u63d2\u4ef6\uff0c\u4f8b\u5982 macvlan\u3001vlan\u3001host-device \u7b49\u63d2\u4ef6\u63d0\u4f9b\u96c6\u7fa4\u7ea7\u522b\u7684 IPAM \u80fd\u529b\uff0c \u5176\u4ed6\u7f51\u7edc\u63d2\u4ef6\u4e5f\u53ef\u4ee5\u4f7f\u7528\u5230 Kube-OVN \u4e2d\u5b50\u7f51\u4ee5\u53ca\u56fa\u5b9a IP \u529f\u80fd\u3002</p> <p>\u540c\u65f6 Kube-OVN \u4e5f\u652f\u6301\u591a\u5757\u7f51\u5361\u5747\u4e3a Kube-OVN \u7c7b\u578b\u7f51\u5361\u60c5\u51b5\u4e0b\u7684\u5730\u5740\u7ba1\u7406\u3002</p>"},{"location":"advance/multi-nic/#_2","title":"\u5de5\u4f5c\u539f\u7406","text":"<p>\u901a\u8fc7\u4f7f\u7528 Multus CNI, \u6211\u4eec\u53ef\u4ee5\u7ed9\u4e00\u4e2a Pod \u6dfb\u52a0\u591a\u5757\u4e0d\u540c\u7f51\u7edc\u7684\u7f51\u5361\u3002 \u7136\u800c\u6211\u4eec\u4ecd\u7136\u7f3a\u4e4f\u5bf9\u96c6\u7fa4\u8303\u56f4\u5185\u4e0d\u540c\u7f51\u7edc\u7684 IP \u5730\u5740\u8fdb\u884c\u7ba1\u7406\u7684\u80fd\u529b\u3002\u5728 Kube-OVN \u4e2d\uff0c\u6211\u4eec\u5df2\u7ecf\u80fd\u591f\u901a\u8fc7 Subnet \u548c IP \u7684 CRD \u6765\u8fdb\u884c IP \u7684\u9ad8\u7ea7\u7ba1\u7406\uff0c \u4f8b\u5982\u5b50\u7f51\u7ba1\u7406\uff0cIP \u9884\u7559\uff0c\u968f\u673a\u5206\u914d\uff0c\u56fa\u5b9a\u5206\u914d\u7b49\u3002\u73b0\u5728\u6211\u4eec\u5bf9\u5b50\u7f51\u8fdb\u884c\u6269\u5c55\uff0c\u6765\u63a5\u5165\u5176\u4ed6\u4e0d\u540c\u7684\u7f51\u7edc\u63d2\u4ef6\uff0c\u4f7f\u5f97\u5176\u4ed6\u7f51\u7edc\u63d2\u4ef6\u4e5f\u53ef\u4ee5\u4f7f\u7528 Kube-OVN \u7684 IPAM \u529f\u80fd\u3002</p>"},{"location":"advance/multi-nic/#_3","title":"\u5de5\u4f5c\u6d41\u7a0b","text":"<p>\u4e0a\u56fe\u5c55\u793a\u4e86\u5982\u4f55\u901a\u8fc7 Kube-OVN \u6765\u7ba1\u7406\u5176\u4ed6\u7f51\u7edc\u63d2\u4ef6\u7684 IP \u5730\u5740\u3002\u5176\u4e2d\u5bb9\u5668\u7684 eth0 \u7f51\u5361\u63a5\u5165 OVN \u7f51\u7edc\uff0cnet1 \u7f51\u5361\u63a5\u5165\u5176\u4ed6 CNI \u7f51\u7edc\u3002 net1 \u7f51\u7edc\u7684\u7f51\u7edc\u5b9a\u4e49\u6765\u81ea\u4e8e multus-cni \u4e2d\u7684 NetworkAttachmentDefinition \u8d44\u6e90\u5b9a\u4e49\u3002</p> <p>\u5f53 Pod \u521b\u5efa\u65f6\uff0c<code>kube-ovn-controller</code> \u4f1a\u76d1\u542c\u5230 Pod \u6dfb\u52a0\u4e8b\u4ef6\uff0c\u5e76\u6839\u636e Pod \u4e2d\u7684 annotation \u53bb\u5bfb\u627e\u5230\u5bf9\u5e94\u7684 Subnet \u5e76\u4ece\u4e2d\u8fdb\u884c IP \u7684\u5206\u914d\u548c\u7ba1\u7406\uff0c \u5e76\u5c06 Pod \u6240\u5206\u914d\u5230\u7684\u5730\u5740\u4fe1\u606f\u5199\u56de\u5230 Pod annotation \u4e2d\u3002</p> <p>\u5728\u5bb9\u5668\u6240\u5728\u673a\u5668\u7684 CNI \u53ef\u4ee5\u901a\u8fc7\u5728\u914d\u7f6e\u4e2d\u914d\u7f6e <code>kube-ovn-cni</code> \u4f5c\u4e3a ipam \u63d2\u4ef6, <code>kube-ovn-cni</code> \u5c06\u4f1a\u8bfb\u53d6 Pod annotation \u5e76\u5c06\u5730\u5740\u4fe1\u606f\u901a\u8fc7 CNI \u534f\u8bae\u7684\u6807\u51c6\u683c\u5f0f\u8fd4\u56de\u7ed9\u76f8\u5e94\u7684 CNI \u63d2\u4ef6\u3002</p>"},{"location":"advance/multi-nic/#_4","title":"\u4f7f\u7528\u65b9\u6cd5","text":""},{"location":"advance/multi-nic/#kube-ovn-multus","title":"\u5b89\u88c5 Kube-OVN \u548c Multus","text":"<p>\u8bf7\u53c2\u8003 Kube-OVN \u4e00\u952e\u5b89\u88c5 \u548c Multus how to use \u6765\u5b89\u88c5 Kube-OVN \u548c Multus-CNI\u3002</p>"},{"location":"advance/multi-nic/#cni-ipam","title":"\u4e3a\u5176\u4ed6 CNI \u63d0\u4f9b IPAM","text":"<p>\u6b64\u65f6\u4e3b\u7f51\u5361\u4e3a Kube-OVN \u7c7b\u578b\u7f51\u5361\uff0c\u9644\u5c5e\u7f51\u5361\u4e3a\u5176\u4ed6\u7c7b\u578b CNI\u3002</p>"},{"location":"advance/multi-nic/#networkattachmentdefinition","title":"\u521b\u5efa NetworkAttachmentDefinition","text":"<p>\u8fd9\u91cc\u6211\u4eec\u4f7f\u7528 macvlan \u4f5c\u4e3a\u5bb9\u5668\u7f51\u7edc\u7684\u7b2c\u4e8c\u4e2a\u7f51\u7edc\uff0c\u5e76\u5c06\u5176 ipam \u8bbe\u7f6e\u4e3a <code>kube-ovn</code>\uff1a</p> <pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: macvlan\nnamespace: default\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.0\",\n\"type\": \"macvlan\",\n\"master\": \"eth0\",\n\"mode\": \"bridge\",\n\"ipam\": {\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"macvlan.default\"\n}\n}'\n</code></pre> <ul> <li><code>spec.config.ipam.type</code>: \u9700\u8981\u4e3a <code>kube-ovn</code> \u6765\u8c03\u7528 kube-ovn \u7684\u63d2\u4ef6\u6765\u83b7\u53d6\u5730\u5740\u4fe1\u606f\u3002</li> <li><code>server_socket</code>: Kube-OVN \u901a\u4fe1\u4f7f\u7528\u7684 socket \u6587\u4ef6\u3002 \u9ed8\u8ba4\u4f4d\u7f6e\u4e3a <code>/run/openvswitch/kube-ovn-daemon.sock</code>\u3002</li> <li><code>provider</code>: \u5f53\u524d NetworkAttachmentDefinition \u7684 <code>&lt;name&gt;.&lt;namespace&gt;</code> , Kube-OVN \u5c06\u4f1a\u4f7f\u7528\u8fd9\u4e9b\u4fe1\u606f\u627e\u5230\u5bf9\u5e94\u7684 Subnet \u8d44\u6e90\u3002</li> </ul>"},{"location":"advance/multi-nic/#kube-ovn","title":"\u9644\u5c5e\u7f51\u5361\u4e3a Kube-OVN \u7c7b\u578b\u7f51\u5361","text":"<p>\u6b64\u65f6\u591a\u5757\u7f51\u5361\u5747\u4e3a Kube-OVN \u7c7b\u578b\u7f51\u5361\u3002</p>"},{"location":"advance/multi-nic/#networkattachmentdefinition_1","title":"\u521b\u5efa NetworkAttachmentDefinition","text":"<p>\u5c06 <code>provider</code> \u7684\u540e\u7f00\u8bbe\u7f6e\u4e3a <code>ovn</code>\uff1a</p> <pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: attachnet\nnamespace: default\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.0\",\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"attachnet.default.ovn\"\n}'\n</code></pre> <ul> <li><code>spec.config.type</code>: \u8bbe\u7f6e\u4e3a <code>kube-ovn</code> \u6765\u89e6\u53d1 CNI \u63d2\u4ef6\u4f7f\u7528 Kube-OVN \u5b50\u7f51\u3002</li> <li><code>server_socket</code>: Kube-OVN \u901a\u4fe1\u4f7f\u7528\u7684 socket \u6587\u4ef6\u3002 \u9ed8\u8ba4\u4f4d\u7f6e\u4e3a <code>/run/openvswitch/kube-ovn-daemon.sock</code>\u3002</li> <li><code>provider</code>: \u5f53\u524d NetworkAttachmentDefinition \u7684 <code>&lt;name&gt;.&lt;namespace&gt;.ovn</code> , Kube-OVN \u5c06\u4f1a\u4f7f\u7528\u8fd9\u4e9b\u4fe1\u606f\u627e\u5230\u5bf9\u5e94\u7684 Subnet \u8d44\u6e90\uff0c\u6ce8\u610f\u540e\u7f00\u9700\u8981\u8bbe\u7f6e\u4e3a ovn\u3002</li> </ul>"},{"location":"advance/multi-nic/#kube-ovn-subnet","title":"\u521b\u5efa\u4e00\u4e2a Kube-OVN Subnet","text":"<p>\u521b\u5efa\u4e00\u4e2a Kube-OVN Subnet,\u8bbe\u7f6e\u5bf9\u5e94\u7684 <code>cidrBlock</code> \u548c <code>exclude_ips</code>, <code>provider</code> \u5e94\u8be5\u8bbe\u7f6e\u4e3a\u5bf9\u5e94\u7684 NetworkAttachmentDefinition \u7684 <code>&lt;name&gt;.&lt;namespace&gt;</code>, \u4f8b\u5982\u7528 macvlan \u63d0\u4f9b\u9644\u52a0\u7f51\u5361\uff0c\u521b\u5efa Subnet \u5982\u4e0b\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: macvlan\nspec:\nprotocol: IPv4\nprovider: macvlan.default\ncidrBlock: 172.17.0.0/16\ngateway: 172.17.0.1\nexcludeIps:\n- 172.17.0.0..172.17.0.10\n</code></pre> <p><code>gateway</code>, <code>private</code>, <code>nat</code> \u53ea\u5bf9 <code>provider</code> \u7c7b\u578b\u4e3a ovn \u7684\u7f51\u7edc\u751f\u6548\uff0c\u4e0d\u9002\u7528\u4e8e attachment network\u3002</p> <p>\u5982\u679c\u4ee5 Kube-OVN \u4f5c\u4e3a\u9644\u52a0\u7f51\u5361\uff0c\u5219 <code>provider</code> \u5e94\u8be5\u8bbe\u7f6e\u4e3a\u5bf9\u5e94\u7684 NetworkAttachmentDefinition \u7684 <code>&lt;name&gt;.&lt;namespace&gt;.ovn</code>\uff0c\u5e76\u8981\u4ee5 <code>ovn</code> \u4f5c\u4e3a\u540e\u7f00\u7ed3\u675f\u3002 \u7528 Kube-OVN \u63d0\u4f9b\u9644\u52a0\u7f51\u5361\uff0c\u521b\u5efa Subnet \u793a\u4f8b\u5982\u4e0b\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: attachnet\nspec:\nprotocol: IPv4\nprovider: attachnet.default.ovn\ncidrBlock: 172.17.0.0/16\ngateway: 172.17.0.1\nexcludeIps:\n- 172.17.0.0..172.17.0.10\n</code></pre>"},{"location":"advance/multi-nic/#pod","title":"\u521b\u5efa\u4e00\u4e2a\u591a\u7f51\u7edc\u7684 Pod","text":"<p>\u5bf9\u4e8e\u5730\u5740\u968f\u673a\u5206\u914d\u7684 Pod\uff0c\u53ea\u9700\u8981\u6dfb\u52a0\u5982\u4e0b annotation <code>k8s.v1.cni.cncf.io/networks</code>,\u53d6\u503c\u4e3a\u5bf9\u5e94\u7684 NetworkAttachmentDefinition \u7684 <code>&lt;namespace&gt;/&lt;name&gt;</code>\uff1a</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: samplepod\nnamespace: default\nannotations:\nk8s.v1.cni.cncf.io/networks: default/macvlan\nspec:\ncontainers:\n- name: samplepod\ncommand: [\"/bin/ash\", \"-c\", \"trap : TERM INT; sleep infinity &amp; wait\"]\nimage: docker.io/library/alpine:edge\n</code></pre>"},{"location":"advance/multi-nic/#ip-pod","title":"\u521b\u5efa\u56fa\u5b9a IP \u7684 Pod","text":"<p>\u5bf9\u4e8e\u56fa\u5b9a IP \u7684 Pod\uff0c\u6dfb\u52a0 <code>&lt;networkAttachmentName&gt;.&lt;networkAttachmentNamespace&gt;.kubernetes.io/ip_address</code> annotation\uff1a</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: static-ip\nnamespace: default\nannotations:\nk8s.v1.cni.cncf.io/networks: default/macvlan\novn.kubernetes.io/ip_address: 10.16.0.15\novn.kubernetes.io/mac_address: 00:00:00:53:6B:B6\nmacvlan.default.kubernetes.io/ip_address: 172.17.0.100\nmacvlan.default.kubernetes.io/mac_address: 00:00:00:53:6B:BB\nspec:\ncontainers:\n- name: static-ip\nimage: docker.io/library/nginx:alpine\n</code></pre>"},{"location":"advance/multi-nic/#ip","title":"\u521b\u5efa\u4f7f\u7528\u56fa\u5b9a IP \u7684\u5de5\u4f5c\u8d1f\u8f7d","text":"<p>\u5bf9\u4e8e\u4f7f\u7528 ippool \u7684\u5de5\u4f5c\u8d1f\u8f7d, \u6dfb\u52a0 <code>&lt;networkAttachmentName&gt;.&lt;networkAttachmentNamespace&gt;.kubernetes.io/ip_pool</code> annotations:</p> <pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nnamespace: default\nname: static-workload\nlabels:\napp: static-workload\nspec:\nreplicas: 2\nselector:\nmatchLabels:\napp: static-workload\ntemplate:\nmetadata:\nlabels:\napp: static-workload\nannotations:\nk8s.v1.cni.cncf.io/networks: default/macvlan\novn.kubernetes.io/ip_pool: 10.16.0.15,10.16.0.16,10.16.0.17\nmacvlan.default.kubernetes.io/ip_pool: 172.17.0.200,172.17.0.201,172.17.0.202\nspec:\ncontainers:\n- name: static-workload\nimage: docker.io/library/nginx:alpine\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/nat-policy-rule/","title":"\u9ed8\u8ba4 VPC NAT \u7b56\u7565\u89c4\u5219","text":""},{"location":"advance/nat-policy-rule/#_1","title":"\u7528\u9014","text":"<p>\u9ed8\u8ba4 VPC \u4e0b\u7684 Overlay \u5b50\u7f51\uff0c\u6253\u5f00 <code>natOutgoing</code> \u5f00\u5173\u65f6\uff0cSubnet \u4e0b\u7684\u6240\u6709 Pod \u8bbf\u95ee\u5916\u7f51\u90fd\u9700\u8981\u505a SNAT \u6210\u8282\u70b9\u7684 IP\uff0c\u4f46\u662f\u6709\u4e9b\u573a\u666f\u6211\u4eec\u5e76\u4e0d\u5e0c\u671b\u5b50\u7f51\u5185\u6240\u6709 Pod \u8bbf\u95ee\u5916\u7f51\u90fd\u505a SNAT\u3002</p> <p>\u56e0\u6b64 NAT \u7b56\u7565\u5c31\u662f\u4e3a\u4e86\u63d0\u4f9b\u4e00\u4e2a\u63a5\u53e3\u8ba9\u7528\u6237\u51b3\u5b9a\u5b50\u7f51\u5185\u7684\u54ea\u4e9b CIDR \u6216\u8005 IP \u8bbf\u95ee\u5916\u7f51\u505a SNAT\u3002</p>"},{"location":"advance/nat-policy-rule/#_2","title":"\u4f7f\u7528\u65b9\u6cd5","text":"<p>\u5728 <code>subnet.Spec</code> \u4e2d\u5f00\u542f <code>natOutgoing</code>\u5f00\u5173\uff0c \u5e76\u4e14\u6dfb\u52a0\u5b57\u6bb5 <code>natOutgoingPolicyRules</code> \u5982\u4e0b\uff1a</p> <pre><code>spec:\nnatOutgoing: true\nnatOutgoingPolicyRules:\n- action: forward\nmatch:\nsrcIPs: 10.0.11.0/30,10.0.11.254\n- action: nat\nmatch:\nsrcIPs: 10.0.11.128/26\ndstIPs: 114.114.114.114,8.8.8.8\n</code></pre> <p>\u4ee5\u4e0a\u6848\u4f8b\u8868\u793a\u6709\u4e24\u6761 NAT \u7b56\u7565\u89c4\u5219\uff1a</p> <ol> <li>\u6e90 IP \u662f 10.0.11.0/30 \u6216\u8005 10.0.11.254  \u7684\u62a5\u6587\u8bbf\u95ee\u5916\u7f51\u65f6\u4e0d\u4f1a\u505a SNAT\u3002</li> <li>\u6e90 IP \u662f 10.0.11.128/26 \u5e76\u4e14\u76ee\u7684 IP \u662f 114.114.114.114 \u6216\u8005 8.8.8.8 \u7684\u62a5\u6587\u8bbf\u95ee\u5916\u7f51\u65f6\u4f1a\u505a SNAT\u3002</li> </ol> <p>\u5b57\u6bb5\u63cf\u8ff0\uff1a</p> <p><code>action</code>\uff1a\u6ee1\u8db3 <code>match</code> \u5bf9\u5e94\u6761\u4ef6\u7684\u62a5\u6587\uff0c\u4f1a\u6267\u884c\u7684 action, action \u5206\u4e3a\u4e24\u79cd <code>forward</code> \u548c <code>nat</code> \uff0c<code>forward</code> \u8868\u793a\u62a5\u6587\u51fa\u5916\u7f51\u4e0d\u505a SNAT, <code>nat</code> \u8868\u793a\u62a5\u6587\u51fa\u5916\u7f51\u505a SNAT\u3002 \u6ca1\u6709\u914d\u7f6e natOutgoingPolicyRules \u65f6\uff0c\u9ed8\u8ba4\u60c5\u51b5\u62a5\u6587\u4ecd\u7136\u662f\u505a SNAT\u3002</p> <p><code>match</code>\uff1a\u8868\u793a\u62a5\u6587\u7684\u5339\u914d\u6bb5\uff0c\u5339\u914d\u6bb5\u6709 <code>srcIPs</code> \u548c <code>dstIPs</code>\uff0c \u8fd9\u91cc\u8868\u793a\u4ece\u5b50\u7f51\u5185\u5230\u5916\u7f51\u65b9\u5411\u4e0a\u7684\u62a5\u6587\u7684\u6e90 IP \u548c \u76ee\u7684 IP\u3002<code>match.srcIPs</code> \u548c <code>match.dstIPs</code> \u652f\u6301\u591a\u4e2a CIDR \u548c IP\uff0c\u4e4b\u95f4\u7528\u9017\u53f7\u95f4\u9694\u3002</p> <p>\u5982\u679c\u51fa\u73b0\u591a\u4e2a match \u89c4\u5219\u91cd\u53e0\uff0c\u5219\u6309\u7167 <code>natOutgoingPolicyRules</code> \u6570\u7ec4\u987a\u5e8f\u8fdb\u884c\u5339\u914d\uff0c\u6700\u5148\u88ab\u5339\u914d\u7684 action \u4f1a\u88ab\u6267\u884c\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/node-local-dns/","title":"\u8282\u70b9\u672c\u5730 DNS \u7f13\u5b58\u548c Kube-OVN \u9002\u914d","text":"<p>NodeLocal DNSCache \u662f\u901a\u8fc7\u96c6\u7fa4\u8282\u70b9\u4e0a\u4f5c\u4e3a DaemonSet \u8fd0\u884c DNS \u7f13\u5b58\u6765\u63d0\u9ad8\u96c6\u7fa4 DNS \u6027\u80fd\uff0c\u8be5\u529f\u80fd\u4e5f\u53ef\u4ee5\u548c Kube-OVN \u9002\u914d\u3002</p>"},{"location":"advance/node-local-dns/#dns","title":"\u8282\u70b9\u672c\u5730 DNS \u7f13\u5b58\u90e8\u7f72","text":""},{"location":"advance/node-local-dns/#kubernetes-dns","title":"\u90e8\u7f72 Kubernetes \u7684\u8282\u70b9\u672c\u5730 DNS \u7f13\u5b58","text":"<p>\u8be5\u6b65\u9aa4\u53c2\u8003 Kubernetes \u5b98\u7f51\u914d\u7f6e Nodelocaldnscache\u3002</p> <p>\u4f7f\u7528\u4ee5\u4e0b\u811a\u672c\u90e8\u7f72\uff1a</p> <pre><code>#!bin/bash\n\nlocaldns=169.254.20.10\ndomain=cluster.local\nkubedns=10.96.0.10\n\nwget https://raw.githubusercontent.com/kubernetes/kubernetes/master/cluster/addons/dns/nodelocaldns/nodelocaldns.yaml\nsed -i \"s/__PILLAR__LOCAL__DNS__/$localdns/g; s/__PILLAR__DNS__DOMAIN__/$domain/g; s/,__PILLAR__DNS__SERVER__//g; s/__PILLAR__CLUSTER__DNS__/$kubedns/g\" nodelocaldns.yaml\n\nkubectl apply -f nodelocaldns.yaml\n</code></pre> <p>\u4fee\u6539\u6bcf\u4e2a\u8282\u70b9\u4e0a\u7684 kubelet \u914d\u7f6e\u6587\u4ef6\uff0c\u5c06 <code>/var/lib/kubelet/config.yaml</code> \u4e2d\u7684 clusterDNS \u5b57\u6bb5\u4fee\u6539\u4e3a\u672c\u5730 DNS IP 169.254.20.10\uff0c\u7136\u540e\u91cd\u542f kubelet \u670d\u52a1\u3002</p>"},{"location":"advance/node-local-dns/#kube-ovn-dns","title":"Kube-OVN \u76f8\u5e94 DNS \u914d\u7f6e","text":"<p>\u90e8\u7f72\u597d Kubernetes \u7684 Nodelocal DNScache \u7ec4\u4ef6\u540e\uff0c Kube-OVN \u9700\u8981\u505a\u51fa\u4e0b\u9762\u4fee\u6539\uff1a</p>"},{"location":"advance/node-local-dns/#underlay-subnet-u2o","title":"Underlay Subnet \u5f00\u542f U2O \u5f00\u5173","text":"<p>\u5982\u679c\u662f Underlay Subnet \u9700\u8981\u4f7f\u7528\u672c\u5730 DNS \u529f\u80fd\uff0c\u9700\u8981\u5f00\u542f U2O \u529f\u80fd\uff0c\u5373\u5728 <code>kubectl edit subnet {your subnet}</code> \u4e2d\u914d\u7f6e <code>spec.u2oInterconnection = true</code> , \u5982\u679c\u662f Overlay Subnet \u5219\u4e0d\u9700\u8981\u8fd9\u6b65\u64cd\u4f5c\u3002</p>"},{"location":"advance/node-local-dns/#kube-ovn-controller-dns-ip","title":"\u7ed9 Kube-ovn-controller \u6307\u5b9a\u5bf9\u5e94\u7684\u672c\u5730 DNS IP","text":"<pre><code>kubectl edit deployment kube-ovn-controller -n kube-system\n</code></pre> <p>\u7ed9 <code>spec.template.spec.containers.args</code> \u589e\u52a0\u5b57\u6bb5 <code>--node-local-dns-ip=169.254.20.10</code></p>"},{"location":"advance/node-local-dns/#pod","title":"\u91cd\u5efa\u5df2\u7ecf\u521b\u5efa\u7684 Pod","text":"<p>\u8fd9\u6b65\u539f\u56e0\u662f\u8ba9 Pod \u91cd\u65b0\u751f\u6210 <code>/etc/resolv.conf</code> \u8ba9 nameserver \u6307\u5411\u672c\u5730 DNS IP\uff0c\u5982\u679c\u6ca1\u6709\u91cd\u5efa Pod \u7684 nameserver \u5c06\u4ecd\u7136\u4f7f\u7528\u96c6\u7fa4\u7684 DNS ClusterIP\u3002\u540c\u65f6 u2o \u5f00\u5173\u5982\u679c\u5f00\u542f\u4e5f\u9700\u8981\u91cd\u5efa Pod \u6765\u91cd\u65b0\u751f\u6210 Pod \u7f51\u5173\u3002</p>"},{"location":"advance/node-local-dns/#dns_1","title":"\u9a8c\u8bc1\u8282\u70b9\u672c\u5730 DNS \u7f13\u5b58\u529f\u80fd","text":"<p>\u4ee5\u4e0a\u914d\u7f6e\u5b8c\u6210\u540e\u53ef\u4ee5\u627e\u5230 Pod \u9a8c\u8bc1\u5982\u4e0b\uff0c\u53ef\u4ee5\u770b\u5230 Pod \u7684 DNS \u670d\u52a1\u5668\u662f\u6307\u5411\u672c\u5730 169.254.20.10 \uff0c\u5e76\u6210\u529f\u89e3\u6790\u57df\u540d\uff1a</p> <pre><code># kubectl exec -it pod1 -- nslookup github.com\nServer:         169.254.20.10\nAddress:        169.254.20.10:53\n\n\nName:   github.com\nAddress: 20.205.243.166\n</code></pre> <p>\u4e5f\u53ef\u4ee5\u5728\u8282\u70b9\u6293\u5305\u9a8c\u8bc1\u5982\u4e0b\uff0c\u53ef\u4ee5\u770b\u5230 DNS \u67e5\u8be2\u62a5\u6587\u901a\u8fc7 ovn0 \u7f51\u5361\u5230\u8fbe\u672c\u5730\u7684 DNS \u670d\u52a1\uff0cDNS \u54cd\u5e94\u62a5\u6587\u539f\u8def\u8fd4\u56de:</p> <pre><code># tcpdump -i any port 53\n\n06:20:00.441889 659246098c56_h P   ifindex 17 00:00:00:73:f1:06 ethertype IPv4 (0x0800), length 75: 10.16.0.2.40230 &gt; 169.254.20.10.53: 1291+ A? baidu.com. (27)\n06:20:00.441889 ovn0  In  ifindex 7 00:00:00:50:32:cd ethertype IPv4 (0x0800), length 75: 10.16.0.2.40230 &gt; 169.254.20.10.53: 1291+ A? baidu.com. (27)\n06:20:00.441950 659246098c56_h P   ifindex 17 00:00:00:73:f1:06 ethertype IPv4 (0x0800), length 75: 10.16.0.2.40230 &gt; 169.254.20.10.53: 1611+ AAAA? baidu.com. (27)\n06:20:00.441950 ovn0  In  ifindex 7 00:00:00:50:32:cd ethertype IPv4 (0x0800), length 75: 10.16.0.2.40230 &gt; 169.254.20.10.53: 1611+ AAAA? baidu.com. (27)\n06:20:00.442203 ovn0  Out ifindex 7 00:00:00:52:99:d8 ethertype IPv4 (0x0800), length 145: 169.254.20.10.53 &gt; 10.16.0.2.40230: 1611* 0/1/0 (97)\n06:20:00.442219 659246098c56_h Out ifindex 17 00:00:00:ea:b3:5e ethertype IPv4 (0x0800), length 145: 169.254.20.10.53 &gt; 10.16.0.2.40230: 1611* 0/1/0 (97)\n06:20:00.442273 ovn0  Out ifindex 7 00:00:00:52:99:d8 ethertype IPv4 (0x0800), length 125: 169.254.20.10.53 &gt; 10.16.0.2.40230: 1291* 2/0/0 A 39.156.66.10, A 110.242.68.66 (77)\n06:20:00.442278 659246098c56_h Out ifindex 17 00:00:00:ea:b3:5e ethertype IPv4 (0x0800), length 125: 169.254.20.10.53 &gt; 10.16.0.2.40230: 1291* 2/0/0 A 39.156.66.10, A 110.242.68.66 (77)\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/offload-corigine/","title":"\u82af\u542f\u6e90\u7f51\u5361 Offload \u652f\u6301","text":"<p>Kube-OVN \u5728\u6700\u7ec8\u7684\u6570\u636e\u5e73\u9762\u4f7f\u7528 OVS \u6765\u5b8c\u6210\u6d41\u91cf\u8f6c\u53d1\uff0c\u76f8\u5173\u7684\u6d41\u8868\u5339\u914d\uff0c\u96a7\u9053\u5c01\u88c5\u7b49\u529f\u80fd\u4e3a CPU \u5bc6\u96c6\u578b\uff0c\u5728\u5927\u6d41\u91cf\u4e0b\u4f1a\u6d88\u8017\u5927\u91cf CPU \u8d44\u6e90\u5e76\u5bfc\u81f4 \u5ef6\u8fdf\u4e0a\u5347\u548c\u541e\u5410\u91cf\u4e0b\u964d\u3002\u82af\u542f\u6e90\u7684 Agilio CX \u7cfb\u5217\u667a\u80fd\u7f51\u5361\u53ef\u4ee5\u5c06 OVS \u76f8\u5173\u7684\u64cd\u4f5c\u5378\u8f7d\u5230\u786c\u4ef6\u7f51\u5361\u4e2d\u6267\u884c\u3002 \u8be5\u6280\u672f\u53ef\u4ee5\u5728\u65e0\u9700\u5bf9 OVS \u63a7\u5236\u5e73\u9762\u8fdb\u884c\u4fee\u6539\u7684\u60c5\u51b5\u4e0b\uff0c\u7f29\u77ed\u6570\u636e\u8def\u5f84\uff0c\u907f\u514d\u5bf9\u4e3b\u673a CPU \u8d44\u6e90\u7684\u4f7f\u7528\uff0c\u5927\u5e45\u964d\u4f4e\u5ef6\u8fdf\u5e76\u663e\u8457\u63d0\u5347\u541e\u5410\u91cf\u3002</p> <p></p>"},{"location":"advance/offload-corigine/#_1","title":"\u524d\u7f6e\u6761\u4ef6","text":"<ul> <li>\u82af\u542f\u6e90 Agilio CX \u7cfb\u5217\u7684\u786c\u4ef6\u7f51\u5361\u3002</li> <li>CentOS 8 Stream \u6216\u4e0a\u6e38 Linux 5.7 \u4ee5\u4e0a\u5185\u6838\u652f\u6301\u3002</li> <li>\u7531\u4e8e\u5f53\u524d\u7f51\u5361\u4e0d\u652f\u6301 <code>dp_hash</code> \u548c <code>hash</code> \u64cd\u4f5c\u5378\u8f7d\uff0c\u9700\u5173\u95ed OVN LB \u529f\u80fd\u3002</li> </ul>"},{"location":"advance/offload-corigine/#sr-iov","title":"\u8bbe\u7f6e\u7f51\u5361 SR-IOV \u6a21\u5f0f","text":"<p>\u7528\u6237\u53ef\u53c2\u8003 Agilio Open vSwitch TC User Guide \u83b7\u5f97\u8be5\u7f51\u5361\u4f7f\u7528\u7684\u66f4\u591a\u8be6\u7ec6\u4fe1\u606f\u3002</p> <p>\u4fdd\u5b58\u4e0b\u5217\u811a\u672c\u7528\u4e8e\u540e\u7eed\u6267\u884c\u56fa\u4ef6\u76f8\u5173\u64cd\u4f5c\uff1a</p> <pre><code>#!/bin/bash\nDEVICE=${1}\nDEFAULT_ASSY=scan\nASSY=${2:-${DEFAULT_ASSY}}\nAPP=${3:-flower}\n\nif [ \"x${DEVICE}\" = \"x\" -o ! -e /sys/class/net/${DEVICE} ]; then\necho Syntax: ${0} device [ASSY] [APP]\necho\necho This script associates the TC Offload firmware\n    echo with a Netronome SmartNIC.\n    echo\necho device: is the network device associated with the SmartNIC\n    echo ASSY: defaults to ${DEFAULT_ASSY}\necho APP: defaults to flower. flower-next is supported if updated\n    echo      firmware has been installed.\n    exit 1\nfi\n\n# It is recommended that the assembly be determined by inspection\n# The following code determines the value via the debug interface\nif [ \"${ASSY}x\" = \"scanx\" ]; then\nethtool -W ${DEVICE} 0\nDEBUG=$(ethtool -w ${DEVICE} data /dev/stdout | strings)\nSERIAL=$(echo \"${DEBUG}\" | grep \"^SN:\")\nASSY=$(echo ${SERIAL} | grep -oE AMDA[0-9]{4})\nfi\n\nPCIADDR=$(basename $(readlink -e /sys/class/net/${DEVICE}/device))\nFWDIR=\"/lib/firmware/netronome\"\n\n# AMDA0081 and AMDA0097 uses the same firmware\nif [ \"${ASSY}\" = \"AMDA0081\" ]; then\nif [ ! -e ${FWDIR}/${APP}/nic_AMDA0081.nffw ]; then\nln -sf nic_AMDA0097.nffw ${FWDIR}/${APP}/nic_AMDA0081.nffw\n   fi\nfi\n\nFW=\"${FWDIR}/pci-${PCIADDR}.nffw\"\nln -sf \"${APP}/nic_${ASSY}.nffw\" \"${FW}\"\n\n# insert distro-specific initramfs section here...\n</code></pre> <p>\u5207\u6362\u56fa\u4ef6\u9009\u9879\u5e76\u91cd\u8f7d\u9a71\u52a8\uff1a</p> <pre><code>./agilio-tc-fw-select.sh ens47np0 scan\nrmmod nfp\nmodprobe nfp\n</code></pre> <p>\u68c0\u67e5\u53ef\u7528 VF \u6570\u91cf\uff0c\u5e76\u521b\u5efa VF\uff1a</p> <pre><code># cat /sys/class/net/ens3/device/sriov_totalvfs\n65\n\n# echo 4 &gt; /sys/class/net/ens47/device/sriov_numvfs\n</code></pre>"},{"location":"advance/offload-corigine/#sr-iov-device-plugin","title":"\u5b89\u88c5 SR-IOV Device Plugin","text":"<p>\u7531\u4e8e\u6bcf\u4e2a\u673a\u5668\u7684 VF \u6570\u91cf\u6709\u9650\uff0c\u6bcf\u4e2a\u4f7f\u7528\u52a0\u901f\u7684 Pod \u4f1a\u5360\u7528 VF \u8d44\u6e90\uff0c\u6211\u4eec\u9700\u8981\u4f7f\u7528 SR-IOV Device Plugin \u7ba1\u7406\u76f8\u5e94\u8d44\u6e90\uff0c\u4f7f\u5f97\u8c03\u5ea6\u5668\u77e5\u9053\u5982\u4f55\u6839\u636e \u8d44\u6e90\u8fdb\u884c\u8c03\u5ea6\u3002</p> <p>\u521b\u5efa SR-IOV \u76f8\u5173 Configmap\uff1a</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: sriovdp-config\nnamespace: kube-system\ndata:\nconfig.json: |\n{\n\"resourceList\": [{\n\"resourcePrefix\": \"corigine.com\",\n\"resourceName\": \"agilio_sriov\",\n\"selectors\": {\n\"vendors\": [\"19ee\"],\n\"devices\": [\"6003\"],\n\"drivers\": [\"nfp_netvf\"]\n}\n}\n]\n}\n</code></pre> <p>\u53c2\u8003 SR-IOV \u6587\u6863\u8fdb\u884c\u90e8\u7f72:</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/intel/sriov-network-device-plugin/master/deployments/k8s-v1.16/sriovdp-daemonset.yaml\n</code></pre> <p>\u68c0\u67e5 SR-IOV \u8d44\u6e90\u662f\u5426\u5df2\u7ecf\u6ce8\u518c\u5230 Kubernetes Node \u4e2d\uff1a</p> <pre><code>kubectl describe no containerserver  | grep corigine\n\ncorigine.com/agilio_sriov:  4\ncorigine.com/agilio_sriov:  4\ncorigine.com/agilio_sriov  0           0\n</code></pre>"},{"location":"advance/offload-corigine/#multus-cni","title":"\u5b89\u88c5 Multus-CNI","text":"<p>SR-IOV Device Plugin \u8c03\u5ea6\u65f6\u83b7\u5f97\u7684\u8bbe\u5907 ID \u9700\u8981\u901a\u8fc7 Multus-CNI \u4f20\u9012\u7ed9 Kube-OVN\uff0c\u56e0\u6b64\u9700\u8981\u914d\u7f6e Multus-CNI \u914d\u5408\u5b8c\u6210\u76f8\u5173\u4efb\u52a1\u3002</p> <p>\u53c2\u8003 Multus-CNI \u6587\u6863\u8fdb\u884c\u90e8\u7f72\uff1a</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml\n</code></pre> <p>\u521b\u5efa <code>NetworkAttachmentDefinition</code>\uff1a</p> <pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: default\nnamespace: default\nannotations:\nk8s.v1.cni.cncf.io/resourceName: corigine.com/agilio_sriov\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.1\",\n\"name\": \"kube-ovn\",\n\"plugins\":[\n{\n\"type\":\"kube-ovn\",\n\"server_socket\":\"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"default.default.ovn\"\n},\n{\n\"type\":\"portmap\",\n\"capabilities\":{\n\"portMappings\":true\n}\n}\n]\n}'\n</code></pre> <ul> <li><code>provider</code>: \u683c\u5f0f\u4e3a\u5f53\u524d <code>NetworkAttachmentDefinition</code> \u7684 {name}.{namespace}.ovn\u3002</li> </ul>"},{"location":"advance/offload-corigine/#kube-ovn","title":"Kube-OVN \u4e2d\u5f00\u542f\u5378\u8f7d\u6a21\u5f0f","text":"<p>\u4e0b\u8f7d\u5b89\u88c5\u811a\u672c\uff1a</p> <pre><code>wget https://raw.githubusercontent.com/alauda/kube-ovn/master/dist/images/install.sh\n</code></pre> <p>\u4fee\u6539\u76f8\u5173\u53c2\u6570\uff0c<code>IFACE</code> \u9700\u8981\u4e3a\u7269\u7406\u7f51\u5361\u540d\uff0c\u8be5\u7f51\u5361\u9700\u8981\u6709\u53ef\u8def\u7531 IP\uff1a</p> <pre><code>ENABLE_MIRROR=${ENABLE_MIRROR:-false}\nHW_OFFLOAD=${HW_OFFLOAD:-true}\nENABLE_LB=${ENABLE_LB:-false}\nIFACE=\"ensp01\"\n</code></pre> <p>\u5b89\u88c5 Kube-OVN\uff1a</p> <pre><code>bash install.sh\n</code></pre>"},{"location":"advance/offload-corigine/#vf-pod","title":"\u521b\u5efa\u4f7f\u7528 VF \u7f51\u5361\u7684 Pod","text":"<p>\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b yaml \u683c\u5f0f\u521b\u5efa\u4f7f\u7528 VF \u8fdb\u884c\u7f51\u7edc\u5378\u8f7d\u52a0\u901f\u7684 Pod:</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: nginx\nnamespace: default\nannotations:\nv1.multus-cni.io/default-network: default/default\nspec:\ncontainers:\n- name: nginx\nimage: docker.io/library/nginx:alpine\nresources:\nrequests:\ncorigine.com/agilio_sriov: '1'\nlimits:\ncorigine.com/agilio_sriov: '1'\n</code></pre> <ul> <li><code>v1.multus-cni.io/default-network</code>: \u4e3a\u4e0a\u4e00\u6b65\u9aa4\u4e2d <code>NetworkAttachmentDefinition</code> \u7684 {namespace}/{name}\u3002</li> </ul> <p>\u53ef\u901a\u8fc7\u5728 Pod \u8fd0\u884c\u8282\u70b9\u7684 <code>ovs-ovn</code> \u5bb9\u5668\u4e2d\u8fd0\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u89c2\u5bdf\u5378\u8f7d\u662f\u5426\u6210\u529f\uff1a</p> <pre><code># ovs-appctl dpctl/dump-flows -m type=offloaded\nufid:91cc45de-e7e9-4935-8f82-1890430b0f66, skb_priority(0/0),skb_mark(0/0),ct_state(0/0x23),ct_zone(0/0),ct_mark(0/0),ct_label(0/0x1),recirc_id(0),dp_hash(0/0),in_port(5b45c61b307e_h),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:c5:6d:4e,dst=00:00:00:e7:16:ce),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:941539, bytes:62142230, used:0.260s, offloaded:yes, dp:tc, actions:54235e5753b8_h\nufid:e00768d7-e652-4d79-8182-3291d852b791, skb_priority(0/0),skb_mark(0/0),ct_state(0/0x23),ct_zone(0/0),ct_mark(0/0),ct_label(0/0x1),recirc_id(0),dp_hash(0/0),in_port(54235e5753b8_h),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:e7:16:ce,dst=00:00:00:c5:6d:4e),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:82386659, bytes:115944854173, used:0.260s, offloaded:yes, dp:tc, actions:5b45c61b307e_h\n</code></pre> <p>\u5982\u679c\u6709 <code>offloaded:yes, dp:tc</code> \u5185\u5bb9\u8bc1\u660e\u5378\u8f7d\u6210\u529f\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/offload-mellanox/","title":"Mellanox \u7f51\u5361 Offload \u652f\u6301","text":"<p>Kube-OVN \u5728\u6700\u7ec8\u7684\u6570\u636e\u5e73\u9762\u4f7f\u7528 OVS \u6765\u5b8c\u6210\u6d41\u91cf\u8f6c\u53d1\uff0c\u76f8\u5173\u7684\u6d41\u8868\u5339\u914d\uff0c\u96a7\u9053\u5c01\u88c5\u7b49\u529f\u80fd\u4e3a CPU \u5bc6\u96c6\u578b\uff0c\u5728\u5927\u6d41\u91cf\u4e0b\u4f1a\u6d88\u8017\u5927\u91cf CPU \u8d44\u6e90\u5e76\u5bfc\u81f4 \u5ef6\u8fdf\u4e0a\u5347\u548c\u541e\u5410\u91cf\u4e0b\u964d\u3002Mellanox \u7684 Accelerated Switching And Packet Processing (ASAP\u00b2) \u6280\u672f\u53ef\u4ee5\u5c06 OVS \u76f8\u5173\u7684\u64cd\u4f5c\u5378\u8f7d\u5230\u786c\u4ef6\u7f51\u5361\u5185\u7684 eSwitch \u4e0a\u6267\u884c\u3002\u8be5\u6280\u672f\u53ef\u4ee5\u5728\u65e0\u9700\u5bf9 OVS \u63a7\u5236\u5e73\u9762\u8fdb\u884c\u4fee\u6539\u7684\u60c5\u51b5\u4e0b\uff0c\u7f29\u77ed\u6570\u636e\u8def\u5f84\uff0c\u907f\u514d\u5bf9\u4e3b\u673a CPU \u8d44\u6e90\u7684\u4f7f\u7528\uff0c\u5927\u5e45\u964d\u4f4e\u5ef6\u8fdf\u5e76\u663e\u8457\u63d0\u5347\u541e\u5410\u91cf\u3002</p> <p></p>"},{"location":"advance/offload-mellanox/#_1","title":"\u524d\u7f6e\u6761\u4ef6","text":"<ul> <li>Mellanox CX5/CX6/CX7/BlueField \u7b49\u652f\u6301 ASAP\u00b2 \u7684\u786c\u4ef6\u7f51\u5361\u3002</li> <li>CentOS 8 Stream \u6216\u4e0a\u6e38 Linux 5.7 \u4ee5\u4e0a\u5185\u6838\u652f\u6301\u3002</li> <li>\u7531\u4e8e\u5f53\u524d\u7f51\u5361\u4e0d\u652f\u6301 <code>dp_hash</code> \u548c <code>hash</code> \u64cd\u4f5c\u5378\u8f7d\uff0c\u9700\u5173\u95ed OVN LB \u529f\u80fd\u3002</li> <li>\u4e3a\u4e86\u652f\u6301\u5378\u8f7d\u6a21\u5f0f\uff0c\u7f51\u5361\u4e0d\u80fd\u505a bond\u3002</li> </ul>"},{"location":"advance/offload-mellanox/#sr-iov-device-plugin","title":"\u914d\u7f6e SR-IOV \u548c Device Plugin","text":"<p>Mellanox \u7f51\u5361\u652f\u6301\u4e24\u79cd\u914d\u7f6e offload \u7684\u65b9\u5f0f\uff0c\u4e00\u79cd\u624b\u52a8\u914d\u7f6e\u7f51\u5361 SR-IOV \u548c Device Plugin\uff0c\u53e6\u4e00\u79cd\u901a\u8fc7 sriov-network-operator \u8fdb\u884c\u81ea\u52a8\u914d\u7f6e\u3002</p>"},{"location":"advance/offload-mellanox/#sr-iov-device-plugin_1","title":"\u624b\u52a8\u914d\u7f6e SR-IOV \u548c Device Plugin","text":"<p>\u67e5\u8be2\u7f51\u5361\u7684\u8bbe\u5907 ID\uff0c\u4e0b\u9762\u7684\u4f8b\u5b50\u4e2d\u4e3a <code>42:00.0</code>\uff1a</p> <pre><code># lspci -nn | grep ConnectX-5\n42:00.0 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5] [15b3:1017]\n</code></pre> <p>\u6839\u636e\u8bbe\u5907 ID \u627e\u5230\u5bf9\u5e94\u7f51\u5361\uff1a</p> <pre><code># ls -l /sys/class/net/ | grep 42:00.0\nlrwxrwxrwx. 1 root root 0 Jul 22 23:16 p4p1 -&gt; ../../devices/pci0000:40/0000:40:02.0/0000:42:00.0/net/p4p1\n</code></pre> <p>\u68c0\u67e5\u53ef\u7528 VF \u6570\u91cf\uff1a</p> <pre><code># cat /sys/class/net/p4p1/device/sriov_totalvfs\n8\n</code></pre> <p>\u521b\u5efa VF\uff0c\u603b\u6570\u4e0d\u8981\u8d85\u8fc7\u4e0a\u9762\u67e5\u8be2\u51fa\u7684\u6570\u91cf\uff1a</p> <pre><code># echo '4' &gt; /sys/class/net/p4p1/device/sriov_numvfs\n# ip link show p4p1\n10: p4p1: &lt;NO-CARRIER,BROADCAST,MULTICAST,UP&gt; mtu 1500 qdisc mq state DOWN mode DEFAULT group default qlen 1000\nlink/ether b8:59:9f:c1:ec:12 brd ff:ff:ff:ff:ff:ff\n    vf 0 MAC 00:00:00:00:00:00, spoof checking off, link-state auto, trust off, query_rss off\n    vf 1 MAC 00:00:00:00:00:00, spoof checking off, link-state auto, trust off, query_rss off\n    vf 2 MAC 00:00:00:00:00:00, spoof checking off, link-state auto, trust off, query_rss off\n    vf 3 MAC 00:00:00:00:00:00, spoof checking off, link-state auto, trust off, query_rss off\n# ip link set p4p1 up\n</code></pre> <p>\u627e\u5230\u4e0a\u8ff0 VF \u5bf9\u5e94\u7684\u8bbe\u5907 ID\uff1a</p> <pre><code># lspci -nn | grep ConnectX-5\n42:00.0 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5] [15b3:1017]\n42:00.1 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5] [15b3:1017]\n42:00.2 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n42:00.3 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n42:00.4 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n42:00.5 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n</code></pre> <p>\u5c06 VF \u4ece\u9a71\u52a8\u4e2d\u89e3\u7ed1\uff1a</p> <pre><code>echo 0000:42:00.2 &gt; /sys/bus/pci/drivers/mlx5_core/unbind\necho 0000:42:00.3 &gt; /sys/bus/pci/drivers/mlx5_core/unbind\necho 0000:42:00.4 &gt; /sys/bus/pci/drivers/mlx5_core/unbind\necho 0000:42:00.5 &gt; /sys/bus/pci/drivers/mlx5_core/unbind\n</code></pre> <p>\u5f00\u542f eSwitch \u6a21\u5f0f\uff0c\u5e76\u8bbe\u7f6e\u786c\u4ef6\u5378\u8f7d\uff1a</p> <pre><code>devlink dev eswitch set pci/0000:42:00.0 mode switchdev\nethtool -K enp66s0f0 hw-tc-offload on\n</code></pre> <p>\u91cd\u65b0\u7ed1\u5b9a\u9a71\u52a8\uff0c\u5b8c\u6210 VF \u8bbe\u7f6e\uff1a</p> <pre><code>echo 0000:42:00.2 &gt; /sys/bus/pci/drivers/mlx5_core/bind\necho 0000:42:00.3 &gt; /sys/bus/pci/drivers/mlx5_core/bind\necho 0000:42:00.4 &gt; /sys/bus/pci/drivers/mlx5_core/bind\necho 0000:42:00.5 &gt; /sys/bus/pci/drivers/mlx5_core/bind\n</code></pre> <p><code>NetworkManager</code> \u7684\u4e00\u4e9b\u884c\u4e3a\u53ef\u80fd\u4f1a\u5bfc\u81f4\u9a71\u52a8\u5f02\u5e38\uff0c\u5982\u679c\u5378\u8f7d\u51fa\u73b0\u95ee\u9898\u5efa\u8bae\u5173\u95ed <code>NetworkManager</code> \u518d\u8fdb\u884c\u5c1d\u8bd5\uff1a</p> <pre><code>systemctl stop NetworkManager\nsystemctl disable NetworkManager\n</code></pre> <p>\u7531\u4e8e\u6bcf\u4e2a\u673a\u5668\u7684 VF \u6570\u91cf\u4f18\u5148\uff0c\u6bcf\u4e2a\u4f7f\u7528\u52a0\u901f\u7684 Pod \u4f1a\u5360\u7528 VF \u8d44\u6e90\uff0c\u6211\u4eec\u9700\u8981\u4f7f\u7528 SR-IOV Device Plugin \u7ba1\u7406\u76f8\u5e94\u8d44\u6e90\uff0c\u4f7f\u5f97\u8c03\u5ea6\u5668\u77e5\u9053\u5982\u4f55\u6839\u636e \u8d44\u6e90\u8fdb\u884c\u8c03\u5ea6\u3002</p> <p>\u521b\u5efa SR-IOV \u76f8\u5173 Configmap\uff1a</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: sriovdp-config\nnamespace: kube-system\ndata:\nconfig.json: |\n{\n\"resourceList\": [{\n\"resourcePrefix\": \"mellanox.com\",\n\"resourceName\": \"cx5_sriov_switchdev\",\n\"selectors\": {\n\"vendors\": [\"15b3\"],\n\"devices\": [\"1018\"],\n\"drivers\": [\"mlx5_core\"]\n}\n}\n]\n}\n</code></pre> <p>\u53c2\u8003 SR-IOV \u6587\u6863\u8fdb\u884c\u90e8\u7f72:</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/intel/sriov-network-device-plugin/master/deployments/k8s-v1.16/sriovdp-daemonset.yaml\n</code></pre> <p>\u68c0\u67e5 SR-IOV \u8d44\u6e90\u662f\u5426\u5df2\u7ecf\u6ce8\u518c\u5230 Kubernetes Node \u4e2d\uff1a</p> <pre><code>kubectl describe node kube-ovn-01  | grep mellanox\n\nmellanox.com/cx5_sriov_switchdev:  4\nmellanox.com/cx5_sriov_switchdev:  4\nmellanox.com/cx5_sriov_switchdev  0           0\n</code></pre>"},{"location":"advance/offload-mellanox/#sriov-network-operator-sr-iov-device-plugin","title":"\u4f7f\u7528 sriov-network-operator \u914d\u7f6e SR-IOV \u548c Device Plugin","text":"<p>\u5b89\u88c5 node-feature-discovery \u81ea\u52a8\u68c0\u6d4b\u786c\u4ef6\u7684\u529f\u80fd\u548c\u7cfb\u7edf\u914d\u7f6e:</p> <pre><code>kubectl apply -k https://github.com/kubernetes-sigs/node-feature-discovery/deployment/overlays/default?ref=v0.11.3\n</code></pre> <p>\u6216\u8005\u901a\u8fc7\u4e0b\u9762\u7684\u547d\u4ee4\uff0c\u624b\u52a8\u7ed9\u6709 offload \u80fd\u529b\u7684\u7f51\u5361\u589e\u52a0 annotation:</p> <pre><code>kubectl label nodes [offloadNicNode] feature.node.kubernetes.io/network-sriov.capable=true\n</code></pre> <p>\u514b\u9686\u4ee3\u7801\u4ed3\u5e93\u5e76\u5b89\u88c5 Operator\uff1a</p> <pre><code>git clone --depth=1 https://github.com/kubeovn/sriov-network-operator.git\nkubectl apply -k sriov-network-operator/deploy\n</code></pre> <p>\u68c0\u67e5 Operator \u7ec4\u4ef6\u662f\u5426\u5de5\u4f5c\u6b63\u5e38\uff1a</p> <pre><code># kubectl get -n kube-system all | grep sriov\nNAME                                          READY   STATUS    RESTARTS   AGE\npod/sriov-network-config-daemon-bf9nt         1/1     Running   0          8s\npod/sriov-network-operator-54d7545f65-296gb   1/1     Running   0          10s\n\nNAME                                         DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR                                                 AGE\ndaemonset.apps/sriov-network-config-daemon   1         1         1       1            1           beta.kubernetes.io/os=linux,feature.node.kubernetes.io/network-sriov.capable=true   8s\n\nNAME                                     READY   UP-TO-DATE   AVAILABLE   AGE\ndeployment.apps/sriov-network-operator   1/1     1            1           10s\n\nNAME                                                DESIRED   CURRENT   READY   AGE\nreplicaset.apps/sriov-network-operator-54d7545f65   1         1         1       10s\n</code></pre> <p>\u68c0\u67e5 <code>SriovNetworkNodeState</code>\uff0c\u4e0b\u9762\u4ee5 <code>node1</code> \u8282\u70b9\u4e3a\u4f8b\uff0c\u8be5\u8282\u70b9\u4e0a\u6709\u4e24\u4e2a Mellanox \u7f51\u5361\uff1a</p> <pre><code># kubectl get sriovnetworknodestates.sriovnetwork.openshift.io -n kube-system node1 -o yaml\napiVersion: sriovnetwork.openshift.io/v1\nkind: SriovNetworkNodeState\nspec: ...\nstatus:\n  interfaces:\n  - deviceID: \"1017\"\ndriver: mlx5_core\n    mtu: 1500\npciAddress: \"0000:5f:00.0\"\ntotalvfs: 8\nvendor: \"15b3\"\nlinkSeed: 25000Mb/s\n    linkType: ETH\n    mac: 08:c0:eb:f4:85:bb\n    name: ens41f0np0\n  - deviceID: \"1017\"\ndriver: mlx5_core\n    mtu: 1500\npciAddress: \"0000:5f:00.1\"\ntotalvfs: 8\nvendor: \"15b3\"\nlinkSeed: 25000Mb/s\n    linkType: ETH\n    mac: 08:c0:eb:f4:85:bb\n    name: ens41f1np1\n</code></pre> <p>\u521b\u5efa <code>SriovNetworkNodePolicy</code> \u8d44\u6e90\uff0c\u5e76\u901a\u8fc7 <code>nicSelector</code> \u9009\u62e9\u8981\u7ba1\u7406\u7684\u7f51\u5361\uff1a</p> <pre><code>apiVersion: sriovnetwork.openshift.io/v1\nkind: SriovNetworkNodePolicy\nmetadata:\nname: policy\nnamespace: kube-system\nspec:\nnodeSelector:\nfeature.node.kubernetes.io/network-sriov.capable: \"true\"\neSwitchMode: switchdev\nnumVfs: 3\nnicSelector:\npfNames:\n- ens41f0np0\n- ens41f1np1\nresourceName: cx_sriov_switchdev\n</code></pre> <p>\u518d\u6b21\u68c0\u67e5 <code>SriovNetworkNodeState</code> \u7684 <code>status</code> \u5b57\u6bb5\uff1a</p> <pre><code># kubectl get sriovnetworknodestates.sriovnetwork.openshift.io -n kube-system node1 -o yaml\n\n...\nspec:\n  interfaces:\n  - eSwitchMode: switchdev\n    name: ens41f0np0\n    numVfs: 3\npciAddress: 0000:5f:00.0\n    vfGroups:\n    - policyName: policy\n      vfRange: 0-2\n      resourceName: cx_sriov_switchdev\n  - eSwitchMode: switchdev\n    name: ens41f1np1\n    numVfs: 3\npciAddress: 0000:5f:00.1\n    vfGroups:\n    - policyName: policy\n      vfRange: 0-2\n      resourceName: cx_sriov_switchdev\nstatus:\n  interfaces\n  - Vfs:\n    - deviceID: 1018\ndriver: mlx5_core\n      pciAddress: 0000:5f:00.2\n      vendor: \"15b3\"\n- deviceID: 1018\ndriver: mlx5_core\n      pciAddress: 0000:5f:00.3\n      vendor: \"15b3\"\n- deviceID: 1018\ndriver: mlx5_core\n      pciAddress: 0000:5f:00.4\n      vendor: \"15b3\"\ndeviceID: \"1017\"\ndriver: mlx5_core\n    linkSeed: 25000Mb/s\n    linkType: ETH\n    mac: 08:c0:eb:f4:85:ab\n    mtu: 1500\nname: ens41f0np0\n    numVfs: 3\npciAddress: 0000:5f:00.0\n    totalvfs: 3\nvendor: \"15b3\"\n- Vfs:\n    - deviceID: 1018\ndriver: mlx5_core\n      pciAddress: 0000:5f:00.5\n      vendor: \"15b3\"\n- deviceID: 1018\ndriver: mlx5_core\n      pciAddress: 0000:5f:00.6\n      vendor: \"15b3\"\n- deviceID: 1018\ndriver: mlx5_core\n      pciAddress: 0000:5f:00.7\n      vendor: \"15b3\"\ndeviceID: \"1017\"\ndriver: mlx5_core\n    linkSeed: 25000Mb/s\n    linkType: ETH\n    mac: 08:c0:eb:f4:85:bb\n    mtu: 1500\nname: ens41f1np1\n    numVfs: 3\npciAddress: 0000:5f:00.1\n    totalvfs: 3\nvendor: \"15b3\"\n</code></pre> <p>\u68c0\u67e5 VF \u7684\u72b6\u6001\uff1a</p> <pre><code># lspci -nn | grep ConnectX\n5f:00.0 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5] [15b3:1017]\n5f:00.1 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5] [15b3:1017]\n5f:00.2 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n5f:00.3 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n5f:00.4 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n5f:00.5 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n5f:00.6 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n5f:00.7 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n</code></pre> <p>\u68c0\u67e5 PF \u5de5\u4f5c\u6a21\u5f0f\uff1a</p> <pre><code># cat /sys/class/net/ens41f0np0/compat/devlink/mode\nswitchdev\n</code></pre>"},{"location":"advance/offload-mellanox/#multus-cni","title":"\u5b89\u88c5 Multus-CNI","text":"<p>SR-IOV Device Plugin \u8c03\u5ea6\u65f6\u83b7\u5f97\u7684\u8bbe\u5907 ID \u9700\u8981\u901a\u8fc7 Multus-CNI \u4f20\u9012\u7ed9 Kube-OVN\uff0c\u56e0\u6b64\u9700\u8981\u914d\u7f6e Multus-CNI \u914d\u5408\u5b8c\u6210\u76f8\u5173\u4efb\u52a1\u3002</p> <p>\u53c2\u8003 Multus-CNI \u6587\u6863\u8fdb\u884c\u90e8\u7f72\uff1a</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml\n</code></pre> <p>\u521b\u5efa <code>NetworkAttachmentDefinition</code>\uff1a</p> <pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: default\nnamespace: default\nannotations:\nk8s.v1.cni.cncf.io/resourceName: mellanox.com/cx5_sriov_switchdev\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.1\",\n\"name\": \"kube-ovn\",\n\"plugins\":[\n{\n\"type\":\"kube-ovn\",\n\"server_socket\":\"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"default.default.ovn\"\n},\n{\n\"type\":\"portmap\",\n\"capabilities\":{\n\"portMappings\":true\n}\n}\n]\n}'\n</code></pre> <ul> <li><code>provider</code>: \u683c\u5f0f\u4e3a\u5f53\u524d <code>NetworkAttachmentDefinition</code> \u7684 {name}.{namespace}.ovn\u3002</li> </ul>"},{"location":"advance/offload-mellanox/#kube-ovn","title":"Kube-OVN \u4e2d\u5f00\u542f\u5378\u8f7d\u6a21\u5f0f","text":"<p>\u4e0b\u8f7d\u5b89\u88c5\u811a\u672c\uff1a</p> <pre><code>wget https://raw.githubusercontent.com/alauda/kube-ovn/master/dist/images/install.sh\n</code></pre> <p>\u4fee\u6539\u76f8\u5173\u53c2\u6570\uff0c<code>IFACE</code> \u9700\u8981\u4e3a\u7269\u7406\u7f51\u5361\u540d\uff0c\u8be5\u7f51\u5361\u9700\u8981\u6709\u53ef\u8def\u7531 IP\uff1a</p> <pre><code>ENABLE_MIRROR=${ENABLE_MIRROR:-false}\nHW_OFFLOAD=${HW_OFFLOAD:-true}\nENABLE_LB=${ENABLE_LB:-false}\nIFACE=\"ensp01\"\n</code></pre> <p>\u5b89\u88c5 Kube-OVN\uff1a</p> <pre><code>bash install.sh\n</code></pre>"},{"location":"advance/offload-mellanox/#vf-pod","title":"\u521b\u5efa\u4f7f\u7528 VF \u7f51\u5361\u7684 Pod","text":"<p>\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b yaml \u683c\u5f0f\u521b\u5efa\u4f7f\u7528 VF \u8fdb\u884c\u7f51\u7edc\u5378\u8f7d\u52a0\u901f\u7684 Pod:</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: nginx\nannotations:\nv1.multus-cni.io/default-network: default/default\nspec:\ncontainers:\n- name: nginx\nimage: docker.io/library/nginx:alpine\nresources:\nrequests:\nmellanox.com/cx5_sriov_switchdev: '1'\nlimits:\nmellanox.com/cx5_sriov_switchdev: '1'\n</code></pre> <ul> <li><code>v1.multus-cni.io/default-network</code>: \u4e3a\u4e0a\u4e00\u6b65\u9aa4\u4e2d <code>NetworkAttachmentDefinition</code> \u7684 {namespace}/{name}\u3002</li> </ul> <p>\u53ef\u901a\u8fc7\u5728 Pod \u8fd0\u884c\u8282\u70b9\u7684 <code>ovs-ovn</code> \u5bb9\u5668\u4e2d\u8fd0\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u89c2\u5bdf\u5378\u8f7d\u662f\u5426\u6210\u529f\uff1a</p> <pre><code># ovs-appctl dpctl/dump-flows -m type=offloaded\nufid:91cc45de-e7e9-4935-8f82-1890430b0f66, skb_priority(0/0),skb_mark(0/0),ct_state(0/0x23),ct_zone(0/0),ct_mark(0/0),ct_label(0/0x1),recirc_id(0),dp_hash(0/0),in_port(5b45c61b307e_h),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:c5:6d:4e,dst=00:00:00:e7:16:ce),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:941539, bytes:62142230, used:0.260s, offloaded:yes, dp:tc, actions:54235e5753b8_h\nufid:e00768d7-e652-4d79-8182-3291d852b791, skb_priority(0/0),skb_mark(0/0),ct_state(0/0x23),ct_zone(0/0),ct_mark(0/0),ct_label(0/0x1),recirc_id(0),dp_hash(0/0),in_port(54235e5753b8_h),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:e7:16:ce,dst=00:00:00:c5:6d:4e),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:82386659, bytes:115944854173, used:0.260s, offloaded:yes, dp:tc, actions:5b45c61b307e_h\n</code></pre> <p>\u5982\u679c\u6709 <code>offloaded:yes, dp:tc</code> \u5185\u5bb9\u8bc1\u660e\u5378\u8f7d\u6210\u529f\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/overlay-with-route/","title":"Overlay \u4e0b\u8def\u7531\u65b9\u5f0f\u7f51\u7edc\u6253\u901a","text":"<p>\u5728\u4e00\u4e9b\u573a\u666f\u4e0b\uff0c\u7f51\u7edc\u73af\u5883\u4e0d\u652f\u6301 Underlay \u6a21\u5f0f\uff0c\u4f46\u662f\u4f9d\u7136\u9700\u8981 Pod \u80fd\u548c\u5916\u90e8\u8bbe\u65bd\u76f4\u63a5\u901a\u8fc7 IP \u8fdb\u884c\u4e92\u8bbf\uff0c \u8fd9\u65f6\u5019\u53ef\u4ee5\u4f7f\u7528\u8def\u7531\u65b9\u5f0f\u5c06\u5bb9\u5668\u7f51\u7edc\u548c\u5916\u90e8\u8054\u901a\u3002</p> <p>\u8def\u7531\u6a21\u5f0f\u53ea\u652f\u6301\u9ed8\u8ba4 VPC \u4e0b\u7684 Overlay \u7f51\u7edc\u548c\u5916\u90e8\u6253\u901a\uff0c\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0cPod IP \u4f1a\u76f4\u63a5\u8fdb\u5165\u5e95\u5c42\u7f51\u7edc\uff0c\u5e95\u5c42\u7f51\u7edc\u9700\u8981\u653e\u5f00\u5173\u4e8e\u6e90\u5730\u5740\u548c\u76ee\u5730\u5740\u7684 IP \u68c0\u67e5\u3002</p>"},{"location":"advance/overlay-with-route/#_1","title":"\u524d\u63d0\u6761\u4ef6","text":"<ul> <li>\u6b64\u6a21\u5f0f\u4e0b\uff0c\u4e3b\u673a\u9700\u8981\u5f00\u653e <code>ip_forward</code>\u3002</li> <li>\u68c0\u67e5\u4e3b\u673a iptables \u89c4\u5219\u4e2d\u662f\u5426\u5728 forward \u94fe\u4e2d\u662f\u5426\u6709 Drop \u89c4\u5219\uff0c\u9700\u8981\u653e\u884c\u5bb9\u5668\u76f8\u5173\u6d41\u91cf\u3002</li> <li>\u7531\u4e8e\u53ef\u80fd\u5b58\u5728\u975e\u5bf9\u79f0\u8def\u7531\u7684\u60c5\u51b5\uff0c\u4e3b\u673a\u9700\u653e\u884c ct \u72b6\u6001\u4e3a <code>INVALID</code> \u7684\u6570\u636e\u5305\u3002</li> </ul>"},{"location":"advance/overlay-with-route/#_2","title":"\u8bbe\u7f6e\u6b65\u9aa4","text":"<p>\u5bf9\u4e8e\u9700\u8981\u5bf9\u5916\u76f4\u63a5\u8def\u7531\u7684\u5b50\u7f51\uff0c\u9700\u8981\u5c06\u5b50\u7f51\u7684 <code>natOutgoing</code> \u8bbe\u7f6e\u4e3a <code>false</code>\uff0c\u5173\u95ed nat \u6620\u5c04\uff0c\u4f7f\u5f97 Pod IP \u53ef\u4ee5\u76f4\u63a5\u8fdb\u5165\u5916\u90e8\u7f51\u7edc\u3002</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: routed\nspec:\nprotocol: IPv4\ncidrBlock: 10.166.0.0/16\ndefault: false\nexcludeIps:\n- 10.166.0.1\ngateway: 10.166.0.1\ngatewayType: distributed\nnatOutgoing: false\n</code></pre> <p>\u6b64\u65f6\uff0cPod \u7684\u6570\u636e\u5305\u53ef\u4ee5\u901a\u8fc7\u4e3b\u673a\u8def\u7531\u5230\u8fbe\u5bf9\u7aef\u8282\u70b9\uff0c\u4f46\u662f\u5bf9\u7aef\u8282\u70b9\u8fd8\u4e0d\u77e5\u9053\u56de\u7a0b\u6570\u636e\u5305\u5e94\u8be5\u53d1\u9001\u5230\u54ea\u91cc\uff0c\u9700\u8981\u6dfb\u52a0\u56de\u7a0b\u8def\u7531\u3002</p> <p>\u5982\u679c\u5bf9\u7aef\u4e3b\u673a\u548c\u5bb9\u5668\u6240\u5728\u5bbf\u4e3b\u673a\u5728\u540c\u4e00\u4e2a\u4e8c\u5c42\u7f51\u7edc\uff0c\u6211\u4eec\u53ef\u4ee5\u76f4\u63a5\u5728\u5bf9\u7aef\u4e3b\u673a\u6dfb\u52a0\u9759\u6001\u8def\u7531\u5c06\u5bb9\u5668\u7f51\u7edc\u7684\u4e0b\u4e00\u8df3\u6307\u5411 Kubernetes \u96c6\u7fa4\u5185\u7684\u4efb\u610f\u4e00\u53f0\u673a\u5668\u3002</p> <pre><code>ip route add 10.166.0.0/16 via 192.168.2.10 dev eth0\n</code></pre> <p><code>10.166.0.0/16</code> \u4e3a\u5bb9\u5668\u5b50\u7f51\u7f51\u6bb5\uff0c<code>192.168.2.10</code> \u4e3a Kubernetes \u96c6\u7fa4\u5185\u4efb\u610f\u4e00\u4e2a\u8282\u70b9\u3002</p> <p>\u82e5\u5bf9\u7aef\u4e3b\u673a\u548c\u5bb9\u5668\u6240\u5728\u5bbf\u4e3b\u673a\u4e0d\u5728\u540c\u4e00\u4e2a\u4e8c\u5c42\u7f51\u7edc\uff0c\u5219\u9700\u8981\u5728\u8def\u7531\u5668\u4e0a\u914d\u7f6e\u76f8\u5e94\u7684\u89c4\u5219\uff0c\u901a\u8fc7\u8def\u7531\u5668\u8fdb\u884c\u6253\u901a\u3002</p> <p>\u6ce8\u610f\uff1a \u6307\u5b9a\u67d0\u4e2a\u8282\u70b9 IP \u5b58\u5728\u5355\u70b9\u6545\u969c\u7684\u53ef\u80fd\uff0c\u5982\u679c\u5e0c\u671b\u505a\u5230\u5feb\u901f\u7684\u6545\u969c\u5207\u6362\u53ef\u4ee5\u901a\u8fc7 Keepalived \u7ed9\u591a\u4e2a\u8282\u70b9\u8bbe\u7f6e VIP\uff0c\u540c\u65f6\u5c06\u8def\u7531\u7684\u4e0b\u4e00\u8df3\u6307\u5411 VIP\u3002</p> <p>\u5728\u4e00\u4e9b\u865a\u62df\u5316\u73af\u5883\u4e2d\uff0c\u865a\u62df\u7f51\u7edc\u4f1a\u5c06\u975e\u5bf9\u79f0\u6d41\u91cf\u8bc6\u522b\u4e3a\u975e\u6cd5\u6d41\u91cf\u5e76\u4e22\u5f03\u3002 \u6b64\u65f6\u9700\u8981\u5c06 Subnet \u7684 <code>gatewayType</code> \u8c03\u6574\u4e3a <code>centralized</code>\uff0c\u5e76\u5728\u8def\u7531\u8bbe\u7f6e\u65f6\u5c06\u4e0b\u4e00\u8df3\u8bbe\u7f6e\u4e3a <code>gatewayNode</code> \u8282\u70b9\u7684 IP\u3002</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: routed\nspec:\nprotocol: IPv4\ncidrBlock: 10.166.0.0/16\ndefault: false\nexcludeIps:\n- 10.166.0.1\ngateway: 10.166.0.1\ngatewayType: centralized\ngatewayNode: \"node1\"\nnatOutgoing: false\n</code></pre> <p>\u5982\u679c\u5bf9\u4e8e\u90e8\u5206\u6d41\u91cf\uff08\u5982\u8bbf\u95ee\u5916\u7f51\u7684\u6d41\u91cf\uff09\u4ecd\u7136\u5e0c\u671b\u8fdb\u884c nat \u5904\u7406\uff0c\u8bf7\u53c2\u8003\u9ed8\u8ba4 VPC NAT \u7b56\u7565\u89c4\u5219\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/ovn-eip-fip-snat/","title":"OVN EIP FIP SNAT DNAT \u652f\u6301","text":"<pre><code>graph LR\n\n\npod--&gt;vpc1-subnet--&gt;vpc1--&gt;snat--&gt;lrp--&gt;external-subnet--&gt;gw-node-external-nic</code></pre> <p>Pod \u57fa\u4e8e SNAT \u51fa\u516c\u7f51\u7684\u5927\u81f4\u6d41\u7a0b\uff0c\u6700\u540e\u662f\u7ecf\u8fc7\u7f51\u5173\u8282\u70b9\u7684\u516c\u7f51\u7f51\u5361\u3002</p> <pre><code>graph LR\n\n\npod--&gt;vpc1-subnet--&gt;vpc1--&gt;fip--&gt;lrp--&gt;external-subnet--&gt;local-node-external-nic</code></pre> <p>Pod \u57fa\u4e8e FIP \u51fa\u516c\u7f51\u7684\u5927\u81f4\u6d41\u7a0b\uff0c\u6700\u540e\u53ef\u4ee5\u57fa\u4e8e\u672c\u5730\u8282\u70b9\u7684\u516c\u7f51\u7f51\u5361\u51fa\u516c\u7f51\u3002</p> <p>\u8be5\u529f\u80fd\u6240\u652f\u6301\u7684 CRD \u5728\u4f7f\u7528\u4e0a\u5c06\u548c iptable nat gw \u516c\u7f51\u65b9\u6848\u4fdd\u6301\u57fa\u672c\u4e00\u81f4\u3002</p> <ul> <li>ovn eip: \u7528\u4e8e\u516c\u7f51 ip \u5360\u4f4d\uff0c\u4ece underlay provider network vlan subnet \u4e2d\u5206\u914d</li> <li>ovn fip\uff1a \u4e00\u5bf9\u4e00 dnat snat\uff0c\u4e3a vpc \u5185\u7684 ip \u6216\u8005 vip \u63d0\u4f9b\u516c\u7f51\u76f4\u63a5\u8bbf\u95ee\u80fd\u529b</li> <li>ovn snat\uff1a\u6574\u4e2a\u5b50\u7f51\u6216\u8005\u5355\u4e2a vpc \u5185 ip \u53ef\u4ee5\u57fa\u4e8e snat \u8bbf\u95ee\u516c\u7f51</li> <li>ovn dnat\uff1a\u57fa\u4e8e router lb \u5b9e\u73b0, \u57fa\u4e8e\u516c\u7f51 ip + \u7aef\u53e3 \u76f4\u63a5\u8bbf\u95ee vpc \u5185\u7684 \u4e00\u7ec4 endpoints</li> </ul>"},{"location":"advance/ovn-eip-fip-snat/#1","title":"1. \u90e8\u7f72","text":"<p>\u76ee\u524d\u5141\u8bb8\u6240\u6709\uff08\u9ed8\u8ba4\u4ee5\u53ca\u81ea\u5b9a\u4e49\uff09vpc \u4f7f\u7528\u540c\u4e00\u4e2a provider vlan subnet \u8d44\u6e90\uff0c\u540c\u65f6\u517c\u5bb9\u9ed8\u8ba4 VPC EIP/SNAT\u7684\u573a\u666f\u3002</p> <p>\u7c7b\u4f3c neutron ovn\uff0c\u670d\u52a1\u542f\u52a8\u914d\u7f6e\u4e2d\u9700\u8981\u6307\u5b9a provider network \u76f8\u5173\u7684\u914d\u7f6e\uff0c\u4e0b\u8ff0\u7684\u542f\u52a8\u53c2\u6570\u4e5f\u662f\u4e3a\u4e86\u517c\u5bb9 VPC EIP/SNAT \u7684\u5b9e\u73b0\u3002</p> <p>\u90e8\u7f72\u9636\u6bb5\uff0c\u6839\u636e\u5b9e\u9645\u60c5\u51b5\uff0c\u53ef\u80fd\u9700\u8981\u6307\u5b9a\u9ed8\u8ba4\u516c\u7f51\u903b\u8f91\u4ea4\u6362\u673a\u3002 \u5982\u679c\u5b9e\u9645\u4f7f\u7528\u4e2d\u6ca1\u6709 vlan\uff08\u4f7f\u7528 vlan 0\uff09\uff0c\u90a3\u4e48\u4e0b\u8ff0\u542f\u52a8\u53c2\u6570\u65e0\u9700\u914d\u7f6e\u3002</p> <pre><code># \u90e8\u7f72\u7684\u65f6\u5019\u4f60\u9700\u8981\u53c2\u8003\u4ee5\u4e0a\u573a\u666f\uff0c\u6839\u636e\u5b9e\u9645\u60c5\u51b5\uff0c\u6309\u9700\u6307\u5b9a\u5982\u4e0b\u53c2\u6570\n# 1. kube-ovn-controller \u542f\u52a8\u53c2\u6570\u9700\u8981\u914d\u7f6e\uff1a\n- --external-gateway-vlanid=204\n- --external-gateway-switch=external204\n\n# 2. kube-ovn-cni \u542f\u52a8\u53c2\u6570\u9700\u8981\u914d\u7f6e:\n- --external-gateway-switch=external204 ### \u4ee5\u4e0a\u914d\u7f6e\u90fd\u548c\u4e0b\u9762\u7684\u516c\u7f51\u7f51\u7edc\u914d\u7f6e vlan id \u548c\u8d44\u6e90\u540d\u4fdd\u6301\u4e00\u81f4\uff0c\u76ee\u524d\u4ec5\u652f\u6301\u6307\u5b9a\u4e00\u4e2a underlay \u516c\u7f51\u4f5c\u4e3a\u9ed8\u8ba4\u5916\u90e8\u516c\u7f51\u3002\n</code></pre> <p>\u8be5\u914d\u7f6e\u9879\u7684\u8bbe\u8ba1\u548c\u4f7f\u7528\u4e3b\u8981\u8003\u8651\u4e86\u5982\u4e0b\u56e0\u7d20\uff1a</p> <ul> <li>\u57fa\u4e8e\u8be5\u914d\u7f6e\u9879\u53ef\u4ee5\u5bf9\u63a5\u5230 provider network\uff0cvlan\uff0csubnet \u7684\u8d44\u6e90\u3002</li> <li>\u57fa\u4e8e\u8be5\u914d\u7f6e\u9879\u53ef\u4ee5\u5c06\u9ed8\u8ba4 vpc enable_eip_snat \u529f\u80fd\u5bf9\u63a5\u5230\u5df2\u6709\u7684 vlan\uff0csubnet \u8d44\u6e90\uff0c\u540c\u65f6\u652f\u6301\u516c\u7f51 ip \u7684 ipam\u3002</li> <li>\u5982\u679c\u4ec5\u4f7f\u7528\u9ed8\u8ba4 vpc \u7684 enable_eip_snat \u6a21\u5f0f, \u4e14\u4ec5\u4f7f\u7528\u65e7\u7684\u57fa\u4e8e pod annotaion \u7684 fip snat\uff0c\u90a3\u4e48\u8fd9\u4e2a\u914d\u7f6e\u65e0\u9700\u914d\u7f6e\u3002</li> <li>\u57fa\u4e8e\u8be5\u914d\u7f6e\u53ef\u4ee5\u4e0d\u4f7f\u7528\u9ed8\u8ba4 vpc enable_eip_snat \u6d41\u7a0b\uff0c\u4ec5\u901a\u8fc7\u5bf9\u5e94\u5230 vlan\uff0csubnet \u6d41\u7a0b\uff0c\u53ef\u4ee5\u517c\u5bb9\u4ec5\u81ea\u5b9a\u4e49 vpc \u4f7f\u7528 eip snat \u7684\u4f7f\u7528\u573a\u666f\u3002</li> </ul>"},{"location":"advance/ovn-eip-fip-snat/#11-underlay","title":"1.1 \u51c6\u5907 underlay \u516c\u7f51\u7f51\u7edc","text":"<pre><code># \u51c6\u5907 provider-network\uff0c vlan\uff0c subnet\n# cat 01-provider-network.yaml\napiVersion: kubeovn.io/v1\nkind: ProviderNetwork\nmetadata:\n  name: external204\nspec:\n  defaultInterface: vlan\n# cat 02-vlan.yaml\napiVersion: kubeovn.io/v1\nkind: Vlan\nmetadata:\n  name: vlan204\nspec:\n  id: 204\nprovider: external204\n# cat 03-vlan-subnet.yaml\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n  name: external204\nspec:\n  protocol: IPv4\n  cidrBlock: 10.5.204.0/24\n  gateway: 10.5.204.254\n  vlan: vlan204\n  excludeIps:\n  - 10.5.204.1..10.5.204.100\n</code></pre>"},{"location":"advance/ovn-eip-fip-snat/#12-vpc-eip_snat","title":"1.2 \u9ed8\u8ba4 vpc \u542f\u7528 eip_snat","text":"<pre><code># \u542f\u7528\u9ed8\u8ba4 vpc \u548c\u4e0a\u8ff0 underlay \u516c\u7f51 provider subnet \u4e92\u8054\ncat 00-centralized-external-gw-no-ip.yaml\napiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: ovn-external-gw-config\n  namespace: kube-system\ndata:\n  enable-external-gw: \"true\"\nexternal-gw-nodes: \"pc-node-1,pc-node-2,pc-node-3\"\ntype: \"centralized\" external-gw-nic: \"vlan\" # \u7528\u4e8e\u63a5\u5165 ovs \u516c\u7f51\u7f51\u6865\u7684\u7f51\u5361\nexternal-gw-addr: \"10.5.204.254/24\" # underlay \u7269\u7406\u7f51\u5173\u7684 ip\n</code></pre> <p>\u76ee\u524d\u8be5\u529f\u80fd\u5df2\u652f\u6301\u53ef\u4ee5\u4e0d\u6307\u5b9a lrp ip \u548c mac\uff0c\u5df2\u652f\u6301\u81ea\u52a8\u83b7\u53d6\uff0c\u521b\u5efa lrp \u7c7b\u578b\u7684 ovn eip \u8d44\u6e90\u3002</p> <p>\u5982\u679c\u6307\u5b9a\u4e86\uff0c\u5219\u76f8\u5f53\u4e8e\u6307\u5b9a ip \u521b\u5efa lrp \u7c7b\u578b\u7684 ovn-eip\u3002 \u5f53\u7136\u4e5f\u53ef\u4ee5\u63d0\u524d\u624b\u52a8\u521b\u5efa lrp \u7c7b\u578b\u7684 ovn eip\u3002</p>"},{"location":"advance/ovn-eip-fip-snat/#13-vpc-eip-snat-fip","title":"1.3 \u81ea\u5b9a\u4e49 vpc \u542f\u7528 eip snat fip \u529f\u80fd","text":"<pre><code># cat 00-ns.yml\napiVersion: v1\nkind: Namespace\nmetadata:\n  name: vpc1\n\n# cat 01-vpc-ecmp-enable-external-bfd.yml\nkind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\n  name: vpc1\nspec:\n  namespaces:\n  - vpc1\n  enableExternal: true\n# vpc \u542f\u7528 enableExternal \u4f1a\u81ea\u52a8\u521b\u5efa lrp \u5173\u8054\u5230\u4e0a\u8ff0\u6307\u5b9a\u7684\u516c\u7f51\n\n# cat 02-subnet.yml\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n  name: vpc1-subnet1\nspec:\n  cidrBlock: 192.168.0.0/24\n  default: false\ndisableGatewayCheck: false\ndisableInterConnection: true\nenableEcmp: true\ngatewayNode: \"\"\ngatewayType: distributed\n  #gatewayType: centralized\nnatOutgoing: false\nprivate: false\nprotocol: IPv4\n  provider: ovn\n  vpc: vpc1\n  namespaces:\n  - vpc1\n\n# \u8fd9\u91cc\u5b50\u7f51\u548c\u4e4b\u524d\u4f7f\u7528\u5b50\u7f51\u4e00\u6837\uff0c\u8be5\u529f\u80fd\u5728 subnet \u4e0a\u6ca1\u6709\u65b0\u589e\u5c5e\u6027\uff0c\u6ca1\u6709\u4efb\u4f55\u53d8\u66f4\n</code></pre> <p>\u4ee5\u4e0a\u6a21\u677f\u5e94\u7528\u540e\uff0c\u5e94\u8be5\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\u8d44\u6e90\u5b58\u5728</p> <pre><code># k ko nbctl show vpc1\nrouter 87ad06fd-71d5-4ff8-a1f0-54fa3bba1a7f (vpc1)\nport vpc1-vpc1-subnet1\n        mac: \"00:00:00:ED:8E:C7\"\nnetworks: [\"192.168.0.1/24\"]\nport vpc1-external204\n        mac: \"00:00:00:EF:05:C7\"\nnetworks: [\"10.5.204.105/24\"]\ngateway chassis: [7cedd14f-265b-42e5-ac17-e03e7a1f2342 276baccb-fe9c-4476-b41d-05872a94976d fd9f140c-c45d-43db-a6c0-0d4f8ea298dd]\nnat 21d853b0-f7b4-40bd-9a53-31d2e2745739\n        external ip: \"10.5.204.115\"\nlogical ip: \"192.168.0.0/24\"\ntype: \"snat\"\n</code></pre> <pre><code># k ko nbctl lr-route-list vpc1\nIPv4 Routes\nRoute Table &lt;main&gt;:\n                0.0.0.0/0              10.5.204.254 dst-ip\n# \u76ee\u524d\u8be5\u8def\u7531\u5df2\u81ea\u52a8\u7ef4\u62a4\n</code></pre>"},{"location":"advance/ovn-eip-fip-snat/#2-ovn-eip","title":"2. ovn-eip","text":"<p>\u8be5\u529f\u80fd\u548c iptables-eip \u8bbe\u8ba1\u548c\u4f7f\u7528\u65b9\u5f0f\u57fa\u672c\u4e00\u81f4\uff0covn-eip \u76ee\u524d\u6709\u4e09\u79cd type</p> <ul> <li>nat: \u7528\u4e8e ovn dnat\uff0cfip, snat, \u8fd9\u4e9b nat \u7c7b\u578b\u4f1a\u8bb0\u5f55\u5728 status \u4e2d</li> <li>lrp: Resources connected to the public network from a vpc can be used by nat</li> <li>lsp: \u7528\u4e8e ovn \u57fa\u4e8e bfd \u7684 ecmp \u9759\u6001\u8def\u7531\u573a\u666f\uff0c\u5728\u7f51\u5173\u8282\u70b9\u4e0a\u63d0\u4f9b\u4e00\u4e2a ovs internal port \u4f5c\u4e3a ecmp \u8def\u7531\u7684\u4e0b\u4e00\u8df3</li> </ul> <pre><code>---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: eip-static\nspec:\n  externalSubnet: external204\n  type: nat\n\n# \u52a8\u6001\u5206\u914d\u4e00\u4e2a eip \u8d44\u6e90\uff0c\u8be5\u8d44\u6e90\u9884\u7559\u7528\u4e8e fip \u573a\u666f\n</code></pre>"},{"location":"advance/ovn-eip-fip-snat/#21-ovn-fip-pod-fip","title":"2.1 ovn-fip \u4e3a pod \u7ed1\u5b9a\u4e00\u4e2a fip","text":"<pre><code># k get po -o wide -n vpc1 vpc-1-busybox01\nNAME              READY   STATUS    RESTARTS   AGE     IP            NODE\nvpc-1-busybox01   1/1     Running   0          3d15h   192.168.0.2   pc-node-2\n\n# k get ip vpc-1-busybox01.vpc1\nNAME                   V4IP          V6IP   MAC                 NODE        SUBNET\nvpc-1-busybox01.vpc1   192.168.0.2          00:00:00:0A:DD:27   pc-node-2   vpc1-subnet1\n\n---\n\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: eip-static\nspec:\n  externalSubnet: external204\n  type: nat\n\n---\nkind: OvnFip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: eip-static\nspec:\n  ovnEip: eip-static\n  ipName: vpc-1-busybox01.vpc1  # \u6ce8\u610f\u8fd9\u91cc\u662f ip crd \u7684\u540d\u5b57\uff0c\u5177\u6709\u552f\u4e00\u6027\n</code></pre> <pre><code># k get ofip\nNAME          VPC    V4EIP          V4IP          READY   IPTYPE   IPNAME\neip-for-vip   vpc1   10.5.204.106   192.168.0.3   true    vip      test-fip-vip\neip-static    vpc1   10.5.204.101   192.168.0.2   true             vpc-1-busybox01.vpc1\n# k get ofip eip-static\nNAME         VPC    V4EIP          V4IP          READY   IPTYPE   IPNAME\neip-static   vpc1   10.5.204.101   192.168.0.2   true             vpc-1-busybox01.vpc1\n\n[root@pc-node-1 03-cust-vpc]# ping 10.5.204.101\nPING 10.5.204.101 (10.5.204.101) 56(84) bytes of data.\n64 bytes from 10.5.204.101: icmp_seq=2 ttl=62 time=1.21 ms\n64 bytes from 10.5.204.101: icmp_seq=3 ttl=62 time=0.624 ms\n64 bytes from 10.5.204.101: icmp_seq=4 ttl=62 time=0.368 ms\n^C\n--- 10.5.204.101 ping statistics ---\n4 packets transmitted, 3 received, 25% packet loss, time 3049ms\nrtt min/avg/max/mdev = 0.368/0.734/1.210/0.352 ms\n[root@pc-node-1 03-cust-vpc]#\n\n# \u53ef\u4ee5\u770b\u5230\u5728 node ping \u9ed8\u8ba4 vpc \u4e0b\u7684 pod \u7684\u516c\u7f51 ip \u662f\u80fd\u901a\u7684\n</code></pre> <pre><code># \u8be5\u516c\u7f51 ip \u80fd\u901a\u7684\u5173\u952e\u8d44\u6e90\u4e3b\u8981\u5305\u62ec\u4ee5\u4e0b\u90e8\u5206\n# k ko nbctl show vpc1\nrouter 87ad06fd-71d5-4ff8-a1f0-54fa3bba1a7f (vpc1)\nport vpc1-vpc1-subnet1\n        mac: \"00:00:00:ED:8E:C7\"\nnetworks: [\"192.168.0.1/24\"]\nport vpc1-external204\n        mac: \"00:00:00:EF:05:C7\"\nnetworks: [\"10.5.204.105/24\"]\ngateway chassis: [7cedd14f-265b-42e5-ac17-e03e7a1f2342 276baccb-fe9c-4476-b41d-05872a94976d fd9f140c-c45d-43db-a6c0-0d4f8ea298dd]\nnat 813523e7-c68c-408f-bd8c-cba30cb2e4f4\n        external ip: \"10.5.204.101\"\nlogical ip: \"192.168.0.2\"\ntype: \"dnat_and_snat\"\n</code></pre>"},{"location":"advance/ovn-eip-fip-snat/#22-ovn-fip-vip-fip","title":"2.2 ovn-fip \u4e3a vip \u7ed1\u5b9a\u4e00\u4e2a fip","text":"<p>\u4e3a\u4e86\u4fbf\u4e8e\u4e00\u4e9b vip \u573a\u666f\u7684\u4f7f\u7528\uff0c\u6bd4\u5982 kubevirt \u865a\u62df\u673a\u5185\u90e8\u6211\u53ef\u80fd\u4f1a\u4f7f\u7528\u4e00\u4e9b vip \u63d0\u4f9b\u7ed9 keepalived\uff0ckube-vip \u7b49\u573a\u666f\u6765\u4f7f\u7528\uff0c\u540c\u65f6\u652f\u6301\u516c\u7f51\u8bbf\u95ee\u3002</p> <p>\u90a3\u4e48\u53ef\u4ee5\u57fa\u4e8e fip \u7ed1\u5b9a vpc \u5185\u90e8\u7684 vip \u7684\u65b9\u5f0f\u6765\u63d0\u4f9b vip \u7684\u516c\u7f51\u80fd\u529b\u3002</p> <pre><code># \u5148\u521b\u5efa vip\uff0ceip\uff0c\u518d\u5c06 eip \u7ed1\u5b9a\u5230 vip\n# cat vip.yaml\napiVersion: kubeovn.io/v1\nkind: Vip\nmetadata:\n  name: test-fip-vip\nspec:\n  subnet: vpc1-subnet1\n\n# cat 04-fip.yaml\n---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: eip-for-vip\nspec:\n  externalSubnet: external204\n  type: nat\n\n---\nkind: OvnFip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: eip-for-vip\nspec:\n  ovnEip: eip-for-vip\n  ipType: vip         # \u9ed8\u8ba4\u60c5\u51b5\u4e0b fip \u662f\u9762\u5411 pod ip \u7684\uff0c\u8fd9\u91cc\u9700\u8981\u6807\u6ce8\u6307\u5b9a\u5bf9\u63a5\u5230 vip \u8d44\u6e90\nipName: test-fip-vip\n</code></pre> <pre><code># k get ofip\nNAME          VPC    V4EIP          V4IP          READY   IPTYPE   IPNAME\neip-for-vip   vpc1   10.5.204.106   192.168.0.3   true    vip      test-fip-vip\n\n\n[root@pc-node-1 fip-vip]# ping  10.5.204.106\nPING 10.5.204.106 (10.5.204.106) 56(84) bytes of data.\n64 bytes from 10.5.204.106: icmp_seq=1 ttl=62 time=0.694 ms\n64 bytes from 10.5.204.106: icmp_seq=2 ttl=62 time=0.436 ms\n\n# \u5728 node \u4e0a\u662f ping \u5f97\u901a\u7684\n\n\n# pod \u5185\u90e8\u7684 ip \u4f7f\u7528\u65b9\u5f0f\u5927\u81f4\u5c31\u662f\u5982\u4e0b\u8fd9\u79cd\u60c5\u51b5\n\n[root@pc-node-1 fip-vip]# k -n vpc1 exec -it vpc-1-busybox03 -- bash\n[root@vpc-1-busybox03 /]#\n[root@vpc-1-busybox03 /]#\n[root@vpc-1-busybox03 /]# ip a\n1: lo: &lt;LOOPBACK,UP,LOWER_UP&gt; mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000\nlink/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00\n    inet 127.0.0.1/8 scope host lo\n       valid_lft forever preferred_lft forever\n    inet6 ::1/128 scope host\n       valid_lft forever preferred_lft forever\n1568: eth0@if1569: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1500 qdisc noqueue state UP group default\n    link/ether 00:00:00:56:40:e5 brd ff:ff:ff:ff:ff:ff link-netnsid 0\ninet 192.168.0.5/24 brd 192.168.0.255 scope global eth0\n       valid_lft forever preferred_lft forever\n    inet 192.168.0.3/24 scope global secondary eth0  # \u53ef\u4ee5\u770b\u5230 vip \u7684\u914d\u7f6e\nvalid_lft forever preferred_lft forever\n    inet6 fe80::200:ff:fe56:40e5/64 scope link\n       valid_lft forever preferred_lft forever\n\n[root@vpc-1-busybox03 /]# tcpdump -i eth0 host  192.168.0.3 -netvv\ntcpdump: listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes\n00:00:00:ed:8e:c7 &gt; 00:00:00:56:40:e5, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 62, id 44830, offset 0, flags [DF], proto ICMP (1), length 84)\n10.5.32.51 &gt; 192.168.0.3: ICMP echo request, id 177, seq 1, length 64\n00:00:00:56:40:e5 &gt; 00:00:00:ed:8e:c7, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 43962, offset 0, flags [none], proto ICMP (1), length 84)\n192.168.0.3 &gt; 10.5.32.51: ICMP echo reply, id 177, seq 1, length 64\n\n# pod \u5185\u90e8\u53ef\u4ee5\u6293\u5230 fip \u76f8\u5173\u7684 icmp \u5305\n</code></pre>"},{"location":"advance/ovn-eip-fip-snat/#3-ovn-snat","title":"3. ovn-snat","text":""},{"location":"advance/ovn-eip-fip-snat/#31-ovn-snat-subnet-cidr","title":"3.1 ovn-snat \u5bf9\u5e94\u4e00\u4e2a subnet \u7684 cidr","text":"<p>\u8be5\u529f\u80fd\u548c iptables-snat \u8bbe\u8ba1\u548c\u4f7f\u7528\u65b9\u5f0f\u57fa\u672c\u4e00\u81f4</p> <pre><code># cat 03-subnet-snat.yaml\n---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: snat-for-subnet-in-vpc\nspec:\n  externalSubnet: external204\n  type: nat\n\n---\nkind: OvnSnatRule\napiVersion: kubeovn.io/v1\nmetadata:\n  name: snat-for-subnet-in-vpc\nspec:\n  ovnEip: snat-for-subnet-in-vpc\n  vpcSubnet: vpc1-subnet1 # eip \u5bf9\u5e94\u6574\u4e2a\u7f51\u6bb5\n</code></pre>"},{"location":"advance/ovn-eip-fip-snat/#32-ovn-snat-pod-ip","title":"3.2 ovn-snat \u5bf9\u5e94\u5230\u4e00\u4e2a pod ip","text":"<p>\u8be5\u529f\u80fd\u548c iptables-snat \u8bbe\u8ba1\u548c\u4f7f\u7528\u65b9\u5f0f\u57fa\u672c\u4e00\u81f4</p> <pre><code># cat 03-pod-snat.yaml\n---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: snat-for-pod-vpc-ip\nspec:\n  externalSubnet: external204\n  type: nat\n\n---\nkind: OvnSnatRule\napiVersion: kubeovn.io/v1\nmetadata:\n  name: snat01\nspec:\n  ovnEip: snat-for-pod-vpc-ip\n  ipName: vpc-1-busybox02.vpc1 # eip \u5bf9\u5e94\u5355\u4e2a pod ip\n</code></pre> <p>\u4ee5\u4e0a\u8d44\u6e90\u521b\u5efa\u540e\uff0c\u53ef\u4ee5\u770b\u5230 snat \u516c\u7f51\u529f\u80fd\u4f9d\u8d56\u7684\u5982\u4e0b\u8d44\u6e90\u3002</p> <pre><code># kubectl ko nbctl show vpc1\nrouter 87ad06fd-71d5-4ff8-a1f0-54fa3bba1a7f (vpc1)\nport vpc1-vpc1-subnet1\n        mac: \"00:00:00:ED:8E:C7\"\nnetworks: [\"192.168.0.1/24\"]\nport vpc1-external204\n        mac: \"00:00:00:EF:05:C7\"\nnetworks: [\"10.5.204.105/24\"]\ngateway chassis: [7cedd14f-265b-42e5-ac17-e03e7a1f2342 276baccb-fe9c-4476-b41d-05872a94976d fd9f140c-c45d-43db-a6c0-0d4f8ea298dd]\nnat 21d853b0-f7b4-40bd-9a53-31d2e2745739\n        external ip: \"10.5.204.115\"\nlogical ip: \"192.168.0.0/24\"\ntype: \"snat\"\nnat da77a11f-c523-439c-b1d1-72c664196a0f\n        external ip: \"10.5.204.116\"\nlogical ip: \"192.168.0.4\"\ntype: \"snat\"\n</code></pre> <pre><code>[root@pc-node-1 03-cust-vpc]# k get po -A -o wide  | grep busy\nvpc1            vpc-1-busybox01                                 1/1     Running   0                3d15h   192.168.0.2   pc-node-2   &lt;none&gt;           &lt;none&gt;\nvpc1            vpc-1-busybox02                                 1/1     Running   0                17h     192.168.0.4   pc-node-1   &lt;none&gt;           &lt;none&gt;\nvpc1            vpc-1-busybox03                                 1/1     Running   0                17h     192.168.0.5   pc-node-1   &lt;none&gt;           &lt;none&gt;\nvpc1            vpc-1-busybox04                                 1/1     Running   0                17h     192.168.0.6   pc-node-3   &lt;none&gt;           &lt;none&gt;\nvpc1            vpc-1-busybox05                                 1/1     Running   0                17h     192.168.0.7   pc-node-1   &lt;none&gt;           &lt;none&gt;\n\n# k exec -it -n vpc1            vpc-1-busybox04   bash\nkubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.\n[root@vpc-1-busybox04 /]#\n[root@vpc-1-busybox04 /]#\n[root@vpc-1-busybox04 /]# ip a\n1: lo: &lt;LOOPBACK,UP,LOWER_UP&gt; mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000\nlink/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00\n    inet 127.0.0.1/8 scope host lo\n       valid_lft forever preferred_lft forever\n    inet6 ::1/128 scope host\n       valid_lft forever preferred_lft forever\n17095: eth0@if17096: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1500 qdisc noqueue state UP group default\n    link/ether 00:00:00:76:94:55 brd ff:ff:ff:ff:ff:ff link-netnsid 0\ninet 192.168.0.6/24 brd 192.168.0.255 scope global eth0\n       valid_lft forever preferred_lft forever\n    inet6 fe80::200:ff:fe76:9455/64 scope link\n       valid_lft forever preferred_lft forever\n[root@vpc-1-busybox04 /]# ping 223.5.5.5\nPING 223.5.5.5 (223.5.5.5) 56(84) bytes of data.\n64 bytes from 223.5.5.5: icmp_seq=1 ttl=114 time=22.2 ms\n64 bytes from 223.5.5.5: icmp_seq=2 ttl=114 time=21.8 ms\n\n[root@pc-node-1 03-cust-vpc]# k exec -it -n vpc1            vpc-1-busybox02   bash\nkubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.\n[root@vpc-1-busybox02 /]#\n[root@vpc-1-busybox02 /]#\n[root@vpc-1-busybox02 /]#\n[root@vpc-1-busybox02 /]# ip a\n1: lo: &lt;LOOPBACK,UP,LOWER_UP&gt; mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000\nlink/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00\n    inet 127.0.0.1/8 scope host lo\n       valid_lft forever preferred_lft forever\n    inet6 ::1/128 scope host\n       valid_lft forever preferred_lft forever\n1566: eth0@if1567: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1500 qdisc noqueue state UP group default\n    link/ether 00:00:00:0b:e9:d0 brd ff:ff:ff:ff:ff:ff link-netnsid 0\ninet 192.168.0.4/24 brd 192.168.0.255 scope global eth0\n       valid_lft forever preferred_lft forever\n    inet6 fe80::200:ff:fe0b:e9d0/64 scope link\n       valid_lft forever preferred_lft forever\n[root@vpc-1-busybox02 /]# ping 223.5.5.5\nPING 223.5.5.5 (223.5.5.5) 56(84) bytes of data.\n64 bytes from 223.5.5.5: icmp_seq=2 ttl=114 time=22.7 ms\n64 bytes from 223.5.5.5: icmp_seq=3 ttl=114 time=22.6 ms\n64 bytes from 223.5.5.5: icmp_seq=4 ttl=114 time=22.1 ms\n^C\n--- 223.5.5.5 ping statistics ---\n4 packets transmitted, 3 received, 25% packet loss, time 3064ms\nrtt min/avg/max/mdev = 22.126/22.518/22.741/0.278 ms\n\n# \u53ef\u4ee5\u770b\u5230\u4e24\u4e2a pod \u53ef\u4ee5\u5206\u522b\u57fa\u4e8e\u8fd9\u4e24\u79cd snat \u8d44\u6e90\u4e0a\u5916\u7f51\n</code></pre>"},{"location":"advance/ovn-eip-fip-snat/#4-ovn-dnat","title":"4. ovn-dnat","text":""},{"location":"advance/ovn-eip-fip-snat/#41-ovn-dnat-pod-dnat","title":"4.1 ovn-dnat \u4e3a pod \u7ed1\u5b9a\u4e00\u4e2a dnat","text":"<pre><code>kind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-static\nspec:\nexternalSubnet: underlay\n---\nkind: OvnDnatRule\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-dnat\nspec:\novnEip: eip-dnat\nipName: vpc-1-busybox01.vpc1 # \u6ce8\u610f\u8fd9\u91cc\u662f pod ip crd \u7684\u540d\u5b57\uff0c\u5177\u6709\u552f\u4e00\u6027\nprotocol: tcp\ninternalPort: \"22\"\nexternalPort: \"22\"\n</code></pre> <p>OvnDnatRule \u7684\u914d\u7f6e\u4e0e IptablesDnatRule \u7c7b\u4f3c</p> <pre><code># kubectl get oeip eip-dnat\nNAME       V4IP        V6IP   MAC                 TYPE   READY\neip-dnat   10.5.49.4          00:00:00:4D:CE:49   dnat   true\n\n# kubectl get odnat\nNAME                   EIP                    PROTOCOL   V4EIP        V4IP           INTERNALPORT   EXTERNALPORT   IPNAME                                READY\neip-dnat               eip-dnat               tcp        10.5.49.4    192.168.0.3    22             22             vpc-1-busybox01.vpc1                  true\n</code></pre>"},{"location":"advance/ovn-eip-fip-snat/#42-ovn-dnat-vip-dnat","title":"4.2 ovn-dnat \u4e3a vip \u7ed1\u5b9a\u4e00\u4e2a dnat","text":"<pre><code>kind: OvnDnatRule\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-dnat\nspec:\nipType: vip  # \u9ed8\u8ba4\u60c5\u51b5\u4e0b dnat \u662f\u9762\u5411 pod ip \u7684\uff0c\u8fd9\u91cc\u9700\u8981\u6807\u6ce8\u6307\u5b9a\u5bf9\u63a5\u5230 vip \u8d44\u6e90\novnEip: eip-dnat\nipName: test-dnat-vip\nprotocol: tcp\ninternalPort: \"22\"\nexternalPort: \"22\"\n</code></pre> <p>OvnDnatRule \u7684\u914d\u7f6e\u4e0e IptablesDnatRule \u7c7b\u4f3c</p> <pre><code># kubectl get vip test-dnat-vip\nNAME            V4IP          PV4IP   MAC                 PMAC   V6IP   PV6IP   SUBNET         READY\ntest-dnat-vip   192.168.0.4           00:00:00:D0:C0:B5                         vpc1-subnet1   true\n\n# kubectl get oeip eip-dnat\nNAME       V4IP        V6IP   MAC                 TYPE   READY\neip-dnat   10.5.49.4          00:00:00:4D:CE:49   dnat   true\n\n# kubectl get odnat eip-dnat \nNAME       EIP        PROTOCOL   V4EIP       V4IP          INTERNALPORT   EXTERNALPORT   IPNAME          READY\neip-dnat   eip-dnat   tcp        10.5.49.4   192.168.0.4   22             22             test-dnat-vip   true\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/ovn-ipsec/","title":"\u4f7f\u7528 IPsec \u52a0\u5bc6\u8282\u70b9\u95f4\u901a\u4fe1","text":"<p>\u8be5\u529f\u80fd\u4ece v1.10.11 \u548c v1.11.4 \u540e\u5f00\u59cb\u652f\u6301\uff0ckernel \u7248\u672c\u81f3\u5c11\u662f 3.10.0 \u4ee5\u4e0a\u3002</p>"},{"location":"advance/ovn-ipsec/#ipsec_1","title":"\u542f\u52a8 IPsec","text":"<p>\u4ece Kube-OVN \u6e90\u7801\u62f7\u8d1d\u811a\u672c ipsec.sh\uff0c\u6267\u884c\u547d\u4ee4\u5982\u4e0b\uff0c\u8be5\u811a\u672c\u4f1a\u8c03\u7528 ovs-pki \u751f\u6210\u548c\u5206\u914d\u52a0\u5bc6\u9700\u8981\u7684\u8bc1\u4e66\uff1a</p> <pre><code>bash ipsec.sh init\n</code></pre> <p>\u6267\u884c\u5b8c\u6bd5\u540e\uff0c\u8282\u70b9\u4e4b\u95f4\u4f1a\u534f\u5546\u4e00\u6bb5\u65f6\u95f4\u5efa\u7acb IPsec \u96a7\u9053\uff0c\u7ecf\u9a8c\u503c\u662f\u5341\u51e0\u79d2\u5230\u4e00\u5206\u949f\u4e4b\u95f4\uff0c\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0b\u547d\u4ee4\u6765\u67e5\u770b IPsec \u72b6\u6001\uff1a</p> <pre><code># sh ipsec.sh status\nPod {ovs-ovn-d7hdt} ipsec status...\nInterface name: ovn-a4718e-0 v1 (CONFIGURED)\nTunnel Type:    geneve\n  Local IP:       172.18.0.2\n  Remote IP:      172.18.0.4\n  Address Family: IPv4\n  SKB mark:       None\n  Local cert:     /etc/ipsec.d/certs/8aebd9df-46ef-47b9-85e3-73e9a765296d-cert.pem\n  Local name:     8aebd9df-46ef-47b9-85e3-73e9a765296d\n  Local key:      /etc/ipsec.d/private/8aebd9df-46ef-47b9-85e3-73e9a765296d-privkey.pem\n  Remote cert:    None\n  Remote name:    a4718e55-5b85-4f46-90e6-63527d080590\n  CA cert:        /etc/ipsec.d/cacerts/cacert.pem\n  PSK:            None\n  Custom Options: {}\nOfport:         2\nCFM state:      Disabled\nKernel policies installed:\n  src 172.18.0.2/32 dst 172.18.0.4/32 proto udp sport 6081\nsrc 172.18.0.2/32 dst 172.18.0.4/32 proto udp sport 6081\nsrc 172.18.0.2/32 dst 172.18.0.4/32 proto udp dport 6081\nsrc 172.18.0.2/32 dst 172.18.0.4/32 proto udp dport 6081\nKernel security associations installed:\n  sel src 172.18.0.2/32 dst 172.18.0.4/32 proto udp sport 6081\nsel src 172.18.0.4/32 dst 172.18.0.2/32 proto udp dport 6081\nsel src 172.18.0.2/32 dst 172.18.0.4/32 proto udp dport 6081\nsel src 172.18.0.4/32 dst 172.18.0.2/32 proto udp sport 6081\nIPsec connections that are active:\n\n Pod {ovs-ovn-fvbbj} ipsec status...\nInterface name: ovn-8aebd9-0 v1 (CONFIGURED)\nTunnel Type:    geneve\n  Local IP:       172.18.0.4\n  Remote IP:      172.18.0.2\n  Address Family: IPv4\n  SKB mark:       None\n  Local cert:     /etc/ipsec.d/certs/a4718e55-5b85-4f46-90e6-63527d080590-cert.pem\n  Local name:     a4718e55-5b85-4f46-90e6-63527d080590\n  Local key:      /etc/ipsec.d/private/a4718e55-5b85-4f46-90e6-63527d080590-privkey.pem\n  Remote cert:    None\n  Remote name:    8aebd9df-46ef-47b9-85e3-73e9a765296d\n  CA cert:        /etc/ipsec.d/cacerts/cacert.pem\n  PSK:            None\n  Custom Options: {}\nOfport:         1\nCFM state:      Disabled\nKernel policies installed:\n  src 172.18.0.4/32 dst 172.18.0.2/32 proto udp dport 6081\nsrc 172.18.0.4/32 dst 172.18.0.2/32 proto udp dport 6081\nsrc 172.18.0.4/32 dst 172.18.0.2/32 proto udp sport 6081\nsrc 172.18.0.4/32 dst 172.18.0.2/32 proto udp sport 6081\nKernel security associations installed:\n  sel src 172.18.0.4/32 dst 172.18.0.2/32 proto udp dport 6081\nsel src 172.18.0.2/32 dst 172.18.0.4/32 proto udp sport 6081\nsel src 172.18.0.4/32 dst 172.18.0.2/32 proto udp sport 6081\nsel src 172.18.0.2/32 dst 172.18.0.4/32 proto udp dport 6081\nIPsec connections that are active:\n</code></pre> <p>\u5efa\u7acb\u5b8c\u6210\u540e\u53ef\u4ee5\u6293\u5305\u89c2\u5bdf\u62a5\u6587\u5df2\u7ecf\u88ab\u52a0\u5bc6\uff1a</p> <pre><code># tcpdump -i eth0 -nel esp\n10:01:40.349896 IP kube-ovn-worker &gt; kube-ovn-control-plane.kind: ESP(spi=0xcc91322a,seq=0x13d0), length 156\n10:01:40.350015 IP kube-ovn-control-plane.kind &gt; kube-ovn-worker: ESP(spi=0xc8df4221,seq=0x1d37), length 156\n</code></pre> <p>\u5f53\u6267\u884c\u5b8c\u811a\u672c\u540e\uff0c\u53ef\u4ee5\u901a\u8fc7\u6267\u884c\u547d\u4ee4\u5173\u95ed IPsec\uff1a</p> <pre><code># bash ipsec.sh stop\n</code></pre> <p>\u6216\u8005\u6267\u884c\u547d\u4ee4\u518d\u6b21\u6253\u5f00\uff1a</p> <pre><code># bash ipsec.sh start\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/ovn-l3-ha-based-ecmp-with-bfd/","title":"OVN SNAT \u57fa\u4e8e ECMP BFD \u9759\u6001\u8def\u7531\u7684 L3 HA \u652f\u6301","text":"<p>\u81ea\u5b9a\u4e49 vpc \u57fa\u4e8e ovn snat \u540e\u57fa\u4e8e ecmp \u9759\u6001\u8def\u7531\u54c8\u5e0c\u5230\u591a\u4e2a gw node ovnext0 \u7f51\u5361\u51fa\u516c\u7f51</p> <ul> <li>\u652f\u6301\u57fa\u4e8e bfd \u7684\u9ad8\u53ef\u7528</li> <li>\u4ec5\u652f\u6301 hash \u8d1f\u8f7d\u5747\u8861</li> </ul> <pre><code>graph LR\n\npod--&gt;vpc-subnet--&gt;vpc--&gt;snat--&gt;ecmp--&gt;external-subnet--&gt;gw-node1-ovnext0--&gt; node1-external-switch\nexternal-subnet--&gt;gw-node2-ovnext0--&gt; node2-external-switch\nexternal-subnet--&gt;gw-node3-ovnext0--&gt; node3-external-switch</code></pre> <p>\u8be5\u529f\u80fd\u7684\u4f7f\u7528\u65b9\u5f0f\u548covn-eip-fip-snat.md \u57fa\u672c\u4e00\u81f4\uff0c\u4e00\u81f4\u7684\u90e8\u5206\u5305\u62ec install.sh \u7684\u90e8\u7f72\u90e8\u5206\uff0cprovider-network\uff0cvlan\uff0csubnet \u7684\u51c6\u5907\u90e8\u5206\u3002</p> <p>\u81f3\u4e8e\u4e0d\u76f8\u540c\u7684\u90e8\u5206\uff0c\u4f1a\u5728\u4ee5\u4e0b\u90e8\u5206\u5177\u4f53\u9610\u8ff0\uff0c\u4e3b\u8981\u5305\u62ec lsp \u7c7b\u578b\u7684 ovn-eip \u7684\u521b\u5efa\uff0c\u4ee5\u53ca\u57fa\u4e8e vpc enable_bfd \u81ea\u52a8\u7ef4\u62a4 bfd \u4ee5\u53ca ecmp \u9759\u6001\u8def\u7531\u3002</p>"},{"location":"advance/ovn-l3-ha-based-ecmp-with-bfd/#1","title":"1. \u90e8\u7f72","text":""},{"location":"advance/ovn-l3-ha-based-ecmp-with-bfd/#11-underlay","title":"1.1 \u51c6\u5907 underlay \u516c\u7f51\u7f51\u7edc","text":""},{"location":"advance/ovn-l3-ha-based-ecmp-with-bfd/#12-vpc-eip_snat","title":"1.2 \u9ed8\u8ba4 vpc \u542f\u7528 eip_snat","text":""},{"location":"advance/ovn-l3-ha-based-ecmp-with-bfd/#13-vpc-eip-snat-fip","title":"1.3 \u81ea\u5b9a\u4e49 vpc \u542f\u7528 eip snat fip \u529f\u80fd","text":"<p>\u4ee5\u4e0a\u90e8\u5206\u548c ovn-eip-fip-snat.md \u5b8c\u5168\u4e00\u81f4\uff0c\u8fd9\u4e9b\u529f\u80fd\u9a8c\u8bc1\u901a\u8fc7\u540e\uff0c\u53ef\u4ee5\u76f4\u63a5\u57fa\u4e8e\u5982\u4e0b\u65b9\u5f0f\uff0c\u5c06 vpc \u5207\u6362\u5230\u57fa\u4e8e ecmp \u7684 bfd \u9759\u6001\u8def\u7531\uff0c\u5f53\u7136\u4e5f\u53ef\u4ee5\u5207\u56de\u3002</p> <p>\u81ea\u5b9a\u4e49 vpc \u4f7f\u7528\u8be5\u529f\u80fd\u4e4b\u524d\uff0c\u9700\u8981\u5148\u63d0\u4f9b\u597d\u7f51\u5173\u8282\u70b9\uff0c\u81f3\u5c11\u9700\u8981\u63d0\u4f9b 2 \u4e2a\u4ee5\u4e0a\u7f51\u5173\u8282\u70b9\uff0c\u6ce8\u610f\u5f53\u524d\u5b9e\u73b0 ovn-eip \u7684\u540d\u5b57\u5fc5\u987b\u548c\u7f51\u5173\u8282\u70b9\u540d\u4fdd\u6301\u4e00\u81f4\uff0c\u76ee\u524d\u6ca1\u6709\u505a\u8be5\u8d44\u6e90\u7684\u81ea\u52a8\u5316\u7ef4\u62a4\u3002</p> <pre><code># cat gw-node-eip.yaml\n---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\nname: pc-node-1\nspec:\nexternalSubnet: external204\ntype: lsp\n\n---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\nname: pc-node-2\nspec:\nexternalSubnet: external204\ntype: lsp\n\n---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\nname: pc-node-3\nspec:\nexternalSubnet: external204\ntype: lsp\n</code></pre> <p>\u7531\u4e8e\u8fd9\u4e2a\u573a\u666f\u76ee\u524d\u8bbe\u8ba1\u4e0a\u662f\u4f9b vpc ecmp \u51fa\u516c\u7f51\u4f7f\u7528\uff0c\u6240\u4ee5\u4ee5\u4e0a\u5728\u6ca1\u6709 vpc \u542f\u7528 bfd \u7684\u65f6\u5019\uff0c\u5373\u4e0d\u5b58\u5728\u5e26\u6709 enable bfd \u6807\u7b7e\u7684 lrp \u7684 ovn eip \u7684\u65f6\u5019\uff0c\u7f51\u5173\u8282\u70b9\u4e0d\u4f1a\u89e6\u53d1\u521b\u5efa\u7f51\u5173\u7f51\u5361\uff0c\u4e5f\u65e0\u6cd5\u6210\u529f\u542f\u52a8\u5bf9\u7aef bfd \u4f1a\u8bdd\u7684\u76d1\u542c\u3002</p>"},{"location":"advance/ovn-l3-ha-based-ecmp-with-bfd/#2-vpc-ecmp-bfd-l3-ha","title":"2. \u81ea\u5b9a\u4e49 vpc \u542f\u7528 ecmp bfd L3 HA \u516c\u7f51\u529f\u80fd","text":"<pre><code># cat 01-vpc-ecmp-enable-external-bfd.yml\nkind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\n  name: vpc1\nspec:\n  namespaces:\n  - vpc1\n  enableExternal: true\nenableBfd: true # bfd \u5f00\u5173\u53ef\u4ee5\u968f\u610f\u5207\u6362\uff0c\u5f00\u8868\u793a\u542f\u7528 bfd ecmp \u8def\u7531\n#enableBfd: false \n\n\n# cat 02-subnet.yml\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n  name: vpc1-subnet1\nspec:\n  cidrBlock: 192.168.0.0/24\n  default: false\ndisableGatewayCheck: false\ndisableInterConnection: true\nenableEcmp: true  # \u53ea\u9700\u5f00\u542f ecmp\ngatewayNode: \"\"\ngatewayType: distributed\n  #gatewayType: centralized\nnatOutgoing: false\nprivate: false\nprotocol: IPv4\n  provider: ovn\n  vpc: vpc1\n  namespaces:\n  - vpc1\n</code></pre> <p>\u4f7f\u7528\u4e0a\u7684\u6ce8\u610f\u70b9:</p> <ol> <li>\u81ea\u5b9a\u4e49 vpc \u4e0b\u7684 ecmp \u53ea\u7528\u9759\u6001 ecmp bfd \u8def\u7531\uff0cvpc enableBfd \u548c subnet enableEcmp \u540c\u65f6\u5f00\u542f\u7684\u60c5\u51b5\u4e0b\u624d\u4f1a\u751f\u6548\uff0c\u624d\u4f1a\u81ea\u52a8\u7ba1\u7406\u9759\u6001 ecmp bfd \u8def\u7531\u3002</li> <li>\u4e0a\u8ff0\u914d\u7f6e\u5173\u95ed\u7684\u60c5\u51b5\u4e0b\uff0c\u4f1a\u81ea\u52a8\u5207\u56de\u5e38\u89c4\u9ed8\u8ba4\u9759\u6001\u8def\u7531\u3002</li> <li>\u9ed8\u8ba4 vpc \u65e0\u6cd5\u4f7f\u7528\u8be5\u529f\u80fd\uff0c\u4ec5\u652f\u6301\u81ea\u5b9a\u4e49 vpc\uff0c\u9ed8\u8ba4 vpc \u6709\u66f4\u590d\u6742\u7684\u7b56\u7565\u8def\u7531\u4ee5\u53ca snat \u8bbe\u8ba1\u3002</li> <li>\u81ea\u5b9a\u4e49 vpc \u7684 subnet \u7684 enableEcmp \u4ec5\u4f7f\u7528\u9759\u6001\u8def\u7531\uff0c\u7f51\u5173\u7c7b\u578b gatewayType \u6ca1\u6709\u4f5c\u7528\u3002</li> <li>\u5f53\u5173\u95ed EnableExternal \u65f6\uff0cvpc \u5185\u65e0\u6cd5\u901a\u5916\u7f51\u3002</li> <li>\u5f53\u5f00\u542f EnableExternal \u65f6\uff0c\u5173\u95ed EnableBfd \u65f6\uff0c\u4f1a\u57fa\u4e8e\u666e\u901a\u9ed8\u8ba4\u8def\u7531\u4e0a\u5916\u7f51\uff0c\u4e0d\u5177\u5907\u9ad8\u53ef\u7528\u3002</li> </ol> <pre><code># \u4e0a\u8ff0\u6a21\u677f\u5e94\u7528\u540e ovn \u903b\u8f91\u5c42\u5e94\u8be5\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\u8d44\u6e90\n# \u67e5\u770b vpc\n# k get vpc\nNAME          ENABLEEXTERNAL   ENABLEBFD   STANDBY   SUBNETS                                NAMESPACES\novn-cluster   true                         true      [\"external204\",\"join\",\"ovn-default\"]\nvpc1          true             true        true      [\"vpc1-subnet1\"]                       [\"vpc1\"]\n\n# \u9ed8\u8ba4 vpc \u672a\u652f\u6301 ENABLEBFD\n# \u81ea\u5b9a\u4e49 vpc \u5df2\u652f\u6301\u4e14\u5df2\u542f\u7528\n\n\n# 1. \u521b\u5efa\u4e86 bfd \u4f1a\u8bdd\n# k ko nbctl list bfd\n_uuid               : be7df545-2c4c-4751-878f-b3507987f050\ndetect_mult         : 3\ndst_ip              : \"10.5.204.121\"\nexternal_ids        : {}\nlogical_port        : vpc1-external204\nmin_rx              : 100\nmin_tx              : 100\noptions             : {}\nstatus              : up\n\n_uuid               : 684c4489-5b59-4693-8d8c-3beab93f8093\ndetect_mult         : 3\ndst_ip              : \"10.5.204.109\"\nexternal_ids        : {}\nlogical_port        : vpc1-external204\nmin_rx              : 100\nmin_tx              : 100\noptions             : {}\nstatus              : up\n\n_uuid               : f0f62077-2ae9-4e79-b4f8-a446ec6e784c\ndetect_mult         : 3\ndst_ip              : \"10.5.204.108\"\nexternal_ids        : {}\nlogical_port        : vpc1-external204\nmin_rx              : 100\nmin_tx              : 100\noptions             : {}\nstatus              : up\n\n### \u6ce8\u610f\u6240\u6709 status \u6b63\u5e38\u90fd\u5e94\u8be5\u662f up \u7684\n\n# 2. \u521b\u5efa\u4e86\u57fa\u4e8e bfd \u7684\u9759\u6001\u8def\u7531\n# k ko nbctl lr-route-list vpc1\nIPv4 Routes\nRoute Table &lt;main&gt;:\n           192.168.0.0/24              10.5.204.108 src-ip ecmp ecmp-symmetric-reply bfd\n           192.168.0.0/24              10.5.204.109 src-ip ecmp ecmp-symmetric-reply bfd\n           192.168.0.0/24              10.5.204.121 src-ip ecmp ecmp-symmetric-reply bfd\n\n# 3. \u9759\u6001\u8def\u7531\u8be6\u60c5\n# k ko nbctl find Logical_Router_Static_Route  policy=src-ip options=ecmp_symmetric_reply=\"true\"\n_uuid               : 3aacb384-d5ee-4b14-aebf-59e8c11717ba\nbfd                 : 684c4489-5b59-4693-8d8c-3beab93f8093\nexternal_ids        : {}\nip_prefix           : \"192.168.0.0/24\"\nnexthop             : \"10.5.204.109\"\noptions             : {ecmp_symmetric_reply=\"true\"}\noutput_port         : []\npolicy              : src-ip\nroute_table         : \"\"\n\n_uuid               : 18bcc585-bc05-430b-925b-ef673c8e1aef\nbfd                 : f0f62077-2ae9-4e79-b4f8-a446ec6e784c\nexternal_ids        : {}\nip_prefix           : \"192.168.0.0/24\"\nnexthop             : \"10.5.204.108\"\noptions             : {ecmp_symmetric_reply=\"true\"}\noutput_port         : []\npolicy              : src-ip\nroute_table         : \"\"\n\n_uuid               : 7d0a4e6b-cde0-4110-8176-fbaf19738498\nbfd                 : be7df545-2c4c-4751-878f-b3507987f050\nexternal_ids        : {}\nip_prefix           : \"192.168.0.0/24\"\nnexthop             : \"10.5.204.121\"\noptions             : {ecmp_symmetric_reply=\"true\"}\noutput_port         : []\npolicy              : src-ip\nroute_table         : \"\"\n</code></pre> <pre><code># \u540c\u65f6\u5728\u7f51\u5173\u8282\u70b9\u90fd\u5e94\u8be5\u5177\u5907\u4ee5\u4e0b\u8d44\u6e90\n\n[root@pc-node-1 ~]# ip netns exec ovnext bash ip a\n/usr/sbin/ip: /usr/sbin/ip: cannot execute binary file\n[root@pc-node-1 ~]#\n[root@pc-node-1 ~]# ip netns exec ovnext ip a\n1: lo: &lt;LOOPBACK,UP,LOWER_UP&gt; mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000\nlink/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00\n    inet 127.0.0.1/8 scope host lo\n       valid_lft forever preferred_lft forever\n    inet6 ::1/128 scope host\n       valid_lft forever preferred_lft forever\n1541: ovnext0: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1400 qdisc noqueue state UNKNOWN group default qlen 1000\nlink/ether 00:00:00:ab:bd:87 brd ff:ff:ff:ff:ff:ff\n    inet 10.5.204.108/24 brd 10.5.204.255 scope global ovnext0\n       valid_lft forever preferred_lft forever\n    inet6 fe80::200:ff:feab:bd87/64 scope link\n       valid_lft forever preferred_lft forever\n[root@pc-node-1 ~]#\n[root@pc-node-1 ~]# ip netns exec ovnext route -n\nKernel IP routing table\nDestination     Gateway         Genmask         Flags Metric Ref    Use Iface\n0.0.0.0         10.5.204.254    0.0.0.0         UG    0      0        0 ovnext0\n10.5.204.0      0.0.0.0         255.255.255.0   U     0      0        0 ovnext0\n\n## \u6ce8\u610f\u4ee5\u4e0a\u5185\u5bb9\u548c\u4e00\u4e2a internal port unerlay \u516c\u7f51 pod \u5185\u90e8\u7684 ns \u5927\u81f4\u662f\u4e00\u81f4\u7684\uff0c\u8fd9\u91cc\u53ea\u662f\u5728\u7f51\u5173\u8282\u70b9\u4e0a\u5355\u72ec\u7ef4\u62a4\u4e86\u4e00\u4e2a ns\n\n[root@pc-node-1 ~]# ip netns exec ovnext bfdd-control status\nThere are 1 sessions:\nSession 1\nid=1 local=10.5.204.108 (p) remote=10.5.204.122 state=Up\n\n## \u8fd9\u91cc\u5373\u662f lrp bfd \u4f1a\u8bdd\u7684\u53e6\u4e00\u7aef\uff0c\u4e5f\u662f lrp ecmp \u7684\u4e0b\u4e00\u8df3\u7684\u5176\u4e2d\u4e00\u4e2a\n\n\n[root@pc-node-1 ~]# ip netns exec ovnext ping -c1 223.5.5.5\nPING 223.5.5.5 (223.5.5.5) 56(84) bytes of data.\n64 bytes from 223.5.5.5: icmp_seq=1 ttl=115 time=21.6 ms\n\n# \u5230\u516c\u7f51\u6ca1\u95ee\u9898\n</code></pre> <p>\u53ef\u4ee5\u5728\u67d0\u4e00\u4e2a\u7f51\u5173\u8282\u70b9\u7684 ovnext ns \u5185\u6293\u5230\u51fa\u53bb\u7684\u5305</p> <pre><code># tcpdump -i ovnext0 host 223.5.5.5 -netvv\ndropped privs to tcpdump\ntcpdump: listening on ovnext0, link-type EN10MB (Ethernet), capture size 262144 bytes\n^C\n0 packets captured\n0 packets received by filter\n0 packets dropped by kernel\n[root@pc-node-1 ~]# exit\n[root@pc-node-1 ~]# ssh pc-node-2\nLast login: Thu Feb 23 09:21:08 2023 from 10.5.32.51\n[root@pc-node-2 ~]# ip netns exec ovnext bash\n[root@pc-node-2 ~]# tcpdump -i ovnext0 host 223.5.5.5 -netvv\ndropped privs to tcpdump\ntcpdump: listening on ovnext0, link-type EN10MB (Ethernet), capture size 262144 bytes\n^C\n0 packets captured\n0 packets received by filter\n0 packets dropped by kernel\n[root@pc-node-2 ~]# exit\n[root@pc-node-2 ~]# logout\nConnection to pc-node-2 closed.\n[root@pc-node-1 ~]# ssh pc-node-3\nLast login: Thu Feb 23 08:32:41 2023 from 10.5.32.51\n[root@pc-node-3 ~]#  ip netns exec ovnext bash\n[root@pc-node-3 ~]# tcpdump -i ovnext0 host 223.5.5.5 -netvv\ndropped privs to tcpdump\ntcpdump: listening on ovnext0, link-type EN10MB (Ethernet), capture size 262144 bytes\n00:00:00:2d:f8:ce &gt; 00:00:00:fd:b2:a4, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 57978, offset 0, flags [DF], proto ICMP (1), length 84)\n10.5.204.102 &gt; 223.5.5.5: ICMP echo request, id 22, seq 71, length 64\n00:00:00:fd:b2:a4 &gt; dc:ef:80:5a:44:1a, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 62, id 57978, offset 0, flags [DF], proto ICMP (1), length 84)\n10.5.204.102 &gt; 223.5.5.5: ICMP echo request, id 22, seq 71, length 64\n^C\n2 packets captured\n2 packets received by filter\n0 packets dropped by kernel\n[root@pc-node-3 ~]#\n\n# \u53ef\u4ee5\u5728\u8be5\u8282\u70b9 down \u6389\u51fa\u53bb\u7684\u7f51\u5361\uff0c\u7136\u540e\u770b pod \u51fa\u53bb\u7684\u5305\u5728\u7f51\u7edc\u4e2d\u65ad\u4e2d\u4f1a\u51fa\u73b0\u51e0\u4e2a\u5305\n# \u4e00\u822c\u90fd\u4f1a\u770b\u5230\u4e22 3 \u4e2a\u5305\n</code></pre>"},{"location":"advance/ovn-l3-ha-based-ecmp-with-bfd/#3-bfd","title":"3. \u5173\u95ed bfd \u6a21\u5f0f","text":"<p>\u5728\u67d0\u4e9b\u573a\u666f\u4e0b\uff0c\u53ef\u80fd\u60f3\u76f4\u63a5\u4f7f\u7528\uff08\u96c6\u4e2d\u5f0f\uff09\u5355\u4e2a\u7f51\u5173\u76f4\u63a5\u51fa\u516c\u7f51\uff0c\u8fd9\u4e2a\u65f6\u5019\u548c\u9ed8\u8ba4 vpc enable_eip_snat \u7684\u4f7f\u7528\u6a21\u5f0f\u662f\u4e00\u81f4\u7684</p> <pre><code># cat 01-vpc-ecmp-enable-external-bfd.yml\nkind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\n  name: vpc2\nspec:\n  namespaces:\n  - vpc2\n  enableExternal: true\n#enableBfd: true\nenableBfd: false\n\n## \u5c06 bfd \u529f\u80fd\u76f4\u63a5\u7981\u7528\u5373\u53ef\n\n# k ko nbctl lr-route-list vpc2\nIPv4 Routes\nRoute Table &lt;main&gt;:\n                0.0.0.0/0              10.5.204.254 dst-ip\n\n# \u5e94\u7528\u540e\u8def\u7531\u4f1a\u5207\u6362\u56de\u6b63\u5e38\u7684\u9ed8\u8ba4\u9759\u6001\u8def\u7531\n# \u540c\u65f6 nbctl list bfd  \u53ef\u4ee5\u770b\u5230 lrp \u5173\u8054\u7684 bfd \u4f1a\u8bdd\u5df2\u7ecf\u79fb\u9664\n# \u800c\u4e14 ovnext ns \u4e2d\u7684\u5bf9\u7aef bfd \u4f1a\u8bdd\u4e5f\u81ea\u52a8\u79fb\u9664\n# \u8be5\u5207\u6362\u8fc7\u7a0b\u4fdd\u6301 vpc subnet \u5185\u4fdd\u6301 ping \u672a\u770b\u5230(\u79d2\u7ea7)\u4e22\u5305\n# \u518d\u5207\u6362\u56de\u53bb \u4e5f\u672a\u770b\u5230(\u79d2\u7ea7)\u4e22\u5305\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/ovn-remote-port-mirroring/","title":"OVN \u6d41\u91cf\u955c\u50cf","text":"<p>\u6b64\u529f\u80fd\u53ef\u4ee5\u5c06\u6307\u5b9a Pod\u3001\u6307\u5b9a\u65b9\u5411\u7684\u6d41\u91cf\uff0c\u901a\u8fc7 GRE/ERSPAN \u5c01\u88c5\u540e\uff0c\u4f20\u8f93\u5230\u8fdc\u7aef\u3002</p> <p>\u6b64\u529f\u80fd\u8981\u6c42 Kube-OVN \u7248\u672c\u4e0d\u4f4e\u4e8e v1.12\u3002</p>"},{"location":"advance/ovn-remote-port-mirroring/#multus-cni","title":"\u90e8\u7f72 Multus-CNI","text":"<p>\u53c2\u8003 Multus-CNI \u6587\u6863 \u90e8\u7f72 Multus\u3002</p>"},{"location":"advance/ovn-remote-port-mirroring/#_1","title":"\u521b\u5efa\u9644\u5c5e\u7f51\u7edc","text":"<p>\u4f7f\u7528\u4ee5\u4e0b\u5185\u5bb9\u521b\u5efa\u9644\u5c5e\u7f51\u7edc\uff1a</p> <pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: attachnet\nnamespace: default\nspec:\nconfig: |\n{\n\"cniVersion\": \"0.3.1\",\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"attachnet.default.ovn\"\n}\n</code></pre> <p>\u5176\u4e2d <code>provider</code> \u5b57\u6bb5\u683c\u5f0f\u4e3a <code>&lt;NAME&gt;.&lt;NAMESPACE&gt;.ovn</code>\u3002</p>"},{"location":"advance/ovn-remote-port-mirroring/#underlay","title":"\u521b\u5efa Underlay \u7f51\u7edc","text":"<p>\u955c\u50cf\u6d41\u91cf\u662f\u5c01\u88c5\u540e\u8fdb\u884c\u4f20\u8f93\u7684\uff0c\u56e0\u6b64\u7528\u4e8e\u4f20\u8f93\u7684\u7f51\u7edc\uff0cMTU \u9700\u8981\u5927\u4e8e\u88ab\u955c\u50cf\u7684 LSP/Pod\u3002\u8fd9\u91cc\u6211\u4eec\u4f7f\u7528 Underlay \u4f5c\u4e3a\u4f20\u8f93\u7f51\u7edc\u3002</p> <p>\u4f7f\u7528\u4ee5\u4e0b\u5185\u5bb9\u521b\u5efa Underlay \u7f51\u7edc\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: ProviderNetwork\nmetadata:\nname: net1\nspec:\ndefaultInterface: eth1\n---\napiVersion: kubeovn.io/v1\nkind: Vlan\nmetadata:\nname: vlan1\nspec:\nid: 0\nprovider: net1\n---\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: subnet1\nspec:\nprotocol: IPv4\ncidrBlock: 172.19.0.0/16\nexcludeIps:\n- 172.19.0.2..172.19.0.20\ngateway: 172.19.0.1\nvlan: vlan1\nprovider: attachnet.default.ovn\n</code></pre> <p>\u5176\u4e2d\uff0c\u5b50\u7f51\u7684 <code>provider</code> \u5fc5\u987b\u4e0e\u9644\u5c5e\u7f51\u7edc\u7684 <code>provider</code> \u76f8\u540c\u3002</p>"},{"location":"advance/ovn-remote-port-mirroring/#pod","title":"\u521b\u5efa\u6d41\u91cf\u63a5\u6536 Pod","text":"<p>\u4f7f\u7528\u4ee5\u4e0b\u5185\u5bb9\u521b\u5efa\u7528\u4e8e\u63a5\u6536\u955c\u50cf\u6d41\u91cf\u7684 Pod\uff1a</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: pod1\nannotations:\nk8s.v1.cni.cncf.io/networks: default/attachnet\nspec:\ncontainers:\n- name: bash\nimage: docker.io/kubeovn/kube-ovn:v1.13.0\nargs:\n- bash\n- -c\n- sleep infinity\nsecurityContext:\nprivileged: true\n</code></pre> <p>\u521b\u5efa\u5b8c\u6210\u540e\uff0c\u67e5\u770b Pod \u7684 IP \u5730\u5740\uff1a</p> <pre><code>$ kubectl get ips | grep pod1\npod1.default                        10.16.0.12   00:00:00:FF:34:24  kube-ovn-worker  ovn-default\npod1.default.attachnet.default.ovn  172.19.0.21  00:00:00:A0:30:68  kube-ovn-worker  subnet1\n</code></pre> <p>\u8bb0\u4f4f\u7b2c\u4e8c\u7f51\u5361\u7684 IP \u5730\u5740 <code>172.19.0.21</code>\u3002</p>"},{"location":"advance/ovn-remote-port-mirroring/#ovn_1","title":"\u521b\u5efa OVN \u6d41\u91cf\u955c\u50cf","text":"<p>\u4f7f\u7528\u4ee5\u4e0b\u547d\u4ee4\u521b\u5efa OVN \u6d41\u91cf\u955c\u50cf\uff1a</p> <pre><code>kubectl ko nbctl mirror-add mirror1 gre 99 from-lport 172.19.0.21\nkubectl ko nbctl lsp-attach-mirror coredns-787d4945fb-gpnkb.kube-system mirror1\n</code></pre> <p>\u5176\u4e2d <code>coredns-787d4945fb-gpnkb.kube-system</code> \u662f OVN LSP \u7684\u540d\u79f0\uff0c\u683c\u5f0f\u901a\u5e38\u4e3a <code>&lt;POD_NAME&gt;.&lt;POD_NAMESPACE&gt;</code>\u3002</p> <p>\u76f8\u5173\u7684 OVN \u547d\u4ee4\u4f7f\u7528\u65b9\u6cd5\u5982\u4e0b\uff1a</p> <pre><code>ovn-nbctl mirror-add &lt;NAME&gt; &lt;TYPE&gt; &lt;INDEX&gt; &lt;FILTER&gt; &lt;IP&gt;\n\nNAME   - add a mirror with given name\nTYPE   - specify TYPE 'gre' or 'erspan'\nINDEX  - specify the tunnel INDEX value\n         (indicates key if GRE, erpsan_idx if ERSPAN)\nFILTER - specify FILTER for mirroring selection\n         ('to-lport' / 'from-lport')\nIP     - specify Sink / Destination i.e. Remote IP\n\novn-nbctl mirror-del [NAME]         remove mirrors\novn-nbctl mirror-list               print mirrors\n\novn-nbctl lsp-attach-mirror PORT MIRROR   attach source PORT to MIRROR\novn-nbctl lsp-detach-mirror PORT MIRROR   detach source PORT from MIRROR\n</code></pre>"},{"location":"advance/ovn-remote-port-mirroring/#pod_1","title":"\u914d\u7f6e\u6d41\u91cf\u63a5\u6536 Pod","text":"<p>\u5728\u524d\u9762\u521b\u5efa\u7684 Pod \u4e2d\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff1a</p> <pre><code>root@pod1:/kube-ovn# ip link add mirror1 type gretap local 172.19.0.21 key 99 dev net1\nroot@pod1:/kube-ovn# ip link set mirror1 up\n</code></pre> <p>\u63a5\u4e0b\u6765\u5c31\u53ef\u4ee5\u5728\u63a5\u6536\u6d41\u91cf\u7684 Pod \u4e2d\u8fdb\u884c\u6293\u5305\u9a8c\u8bc1\uff1a</p> <pre><code>root@pod1:/kube-ovn# tcpdump -i mirror1 -nnve\ntcpdump: listening on mirror1, link-type EN10MB (Ethernet), snapshot length 262144 bytes\n05:13:30.328808 00:00:00:a3:f5:e2 &gt; 00:00:00:97:0f:6e, ethertype ARP (0x0806), length 42: Ethernet (len 6), IPv4 (len 4), Request who-has 10.16.0.7 tell 10.16.0.4, length 28\n05:13:30.559167 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 212: (tos 0x0, ttl 64, id 57364, offset 0, flags [DF], proto UDP (17), length 198)\n10.16.0.4.53 &gt; 10.16.0.6.50472: 34511 NXDomain*- 0/1/1 (170)\n05:13:30.559343 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 212: (tos 0x0, ttl 64, id 57365, offset 0, flags [DF], proto UDP (17), length 198)\n10.16.0.4.53 &gt; 10.16.0.6.45177: 1659 NXDomain*- 0/1/1 (170)\n05:13:30.560625 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 200: (tos 0x0, ttl 64, id 57367, offset 0, flags [DF], proto UDP (17), length 186)\n10.16.0.4.53 &gt; 10.16.0.6.43848: 2636*- 0/1/1 (158)\n05:13:30.562774 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 191: (tos 0x0, ttl 64, id 57368, offset 0, flags [DF], proto UDP (17), length 177)\n10.16.0.4.53 &gt; 10.16.0.6.37755: 48737 NXDomain*- 0/1/1 (149)\n05:13:30.563523 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 187: (tos 0x0, ttl 64, id 57369, offset 0, flags [DF], proto UDP (17), length 173)\n10.16.0.4.53 &gt; 10.16.0.6.53887: 45519 NXDomain*- 0/1/1 (145)\n05:13:30.564940 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 201: (tos 0x0, ttl 64, id 57370, offset 0, flags [DF], proto UDP (17), length 187)\n10.16.0.4.53 &gt; 10.16.0.6.40846: 25745 NXDomain*- 0/1/1 (159)\n05:13:30.565140 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 201: (tos 0x0, ttl 64, id 57371, offset 0, flags [DF], proto UDP (17), length 187)\n10.16.0.4.53 &gt; 10.16.0.6.45214: 61875 NXDomain*- 0/1/1 (159)\n05:13:30.566023 00:00:00:a3:f5:e2 &gt; 00:00:00:55:e4:4e, ethertype IPv4 (0x0800), length 80: (tos 0x0, ttl 64, id 45937, offset 0, flags [DF], proto UDP (17), length 66)\n10.16.0.4.44116 &gt; 172.18.0.1.53: 16025+ [1au] AAAA? alauda.cn. (38)\n</code></pre>"},{"location":"advance/ovn-remote-port-mirroring/#_2","title":"\u6ce8\u610f\u4e8b\u9879","text":"<ol> <li>\u5982\u679c\u4f7f\u7528 ERSPAN \u4f5c\u4e3a\u5c01\u88c5\u534f\u8bae\uff0cOVN \u8282\u70b9\u53ca\u8fdc\u7aef\u8bbe\u5907\u7684 Linux \u5185\u6838\u7248\u672c\u4e0d\u5f97\u4f4e\u4e8e 4.14\u3002\u82e5\u4f7f\u7528 ERSPAN \u4f5c\u4e3a\u5c01\u88c5\u534f\u8bae\u4e14\u4f7f\u7528 IPv6 \u4f5c\u4e3a\u4f20\u8f93\u7f51\u7edc\uff0cLinux \u5185\u6838\u7248\u672c\u4e0d\u5f97\u4f4e\u4e8e 4.16\u3002</li> <li>\u955c\u50cf\u6d41\u91cf\u7684\u4f20\u8f93\u662f\u5355\u5411\u7684\uff0c\u53ea\u9700\u4fdd\u8bc1 OVN \u8282\u70b9\u80fd\u591f\u8bbf\u95ee\u8fdc\u7aef\u8bbe\u5907\u5373\u53ef\u3002</li> </ol> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/performance-tuning/","title":"\u6027\u80fd\u8c03\u4f18","text":"<p>\u4e3a\u4e86\u4fdd\u6301\u5b89\u88c5\u7684\u7b80\u5355\u548c\u529f\u80fd\u7684\u5b8c\u5907\uff0cKube-OVN \u7684\u9ed8\u8ba4\u5b89\u88c5\u811a\u672c\u5e76\u6ca1\u6709\u5bf9\u6027\u80fd\u9488\u5bf9\u6027\u7684\u4f18\u5316\u3002\u5982\u679c\u5e94\u7528\u5bf9\u5ef6\u8fdf\u548c\u541e\u5410\u91cf\u654f\u611f\uff0c \u7ba1\u7406\u5458\u53ef\u4ee5\u901a\u8fc7\u672c\u6587\u6863\u5bf9\u6027\u80fd\u8fdb\u884c\u9488\u5bf9\u6027\u4f18\u5316\u3002</p> <p>\u793e\u533a\u4f1a\u4e0d\u65ad\u8fed\u4ee3\u63a7\u5236\u9762\u677f\u548c\u4f18\u5316\u9762\u7684\u6027\u80fd\uff0c\u90e8\u5206\u901a\u7528\u6027\u80fd\u4f18\u5316\u5df2\u7ecf\u96c6\u6210\u5230\u6700\u65b0\u7248\u672c\uff0c\u5efa\u8bae\u4f7f\u7528\u6700\u65b0\u7248\u672c\u83b7\u5f97\u66f4\u597d\u7684\u9ed8\u8ba4\u6027\u80fd\u3002</p> <p>\u66f4\u591a\u5173\u4e8e\u6027\u80fd\u4f18\u5316\u7684\u8fc7\u7a0b\u548c\u65b9\u6cd5\u8bba\uff0c\u53ef\u4ee5\u89c2\u770b\u89c6\u9891\u5206\u4eab\uff1aKube-OVN \u5bb9\u5668\u6027\u80fd\u4f18\u5316\u4e4b\u65c5\u3002</p>"},{"location":"advance/performance-tuning/#_2","title":"\u57fa\u51c6\u6d4b\u8bd5","text":"<p>\u7531\u4e8e\u8f6f\u786c\u4ef6\u73af\u5883\u7684\u5dee\u5f02\u6781\u5927\uff0c\u8fd9\u91cc\u63d0\u4f9b\u7684\u6027\u80fd\u6d4b\u8bd5\u6570\u636e\u53ea\u80fd\u4f5c\u4e3a\u53c2\u8003\uff0c\u5b9e\u9645\u6d4b\u8bd5\u7ed3\u679c\u4f1a\u548c\u672c\u6587\u6863\u4e2d\u7684\u7ed3\u679c\u5b58\u5728\u8f83\u5927\u5dee\u5f02\u3002 \u5efa\u8bae\u6bd4\u8f83\u4f18\u5316\u524d\u540e\u7684\u6027\u80fd\u6d4b\u8bd5\u7ed3\u679c\uff0c\u548c\u5bbf\u4e3b\u673a\u7f51\u7edc\u548c\u5bb9\u5668\u7f51\u7edc\u7684\u6027\u80fd\u6bd4\u8f83\u3002</p>"},{"location":"advance/performance-tuning/#overlay","title":"Overlay \u4f18\u5316\u524d\u540e\u6027\u80fd\u5bf9\u6bd4","text":"<p>\u73af\u5883\u4fe1\u606f\uff1a</p> <ul> <li>Kubernetes: 1.22.0</li> <li>OS: CentOS 7</li> <li>Kube-OVN: 1.8.0 Overlay \u6a21\u5f0f</li> <li>CPU: Intel(R) Xeon(R) E-2278G</li> <li>Network: 2*10Gbps, xmit_hash_policy=layer3+4</li> </ul> <p>\u6211\u4eec\u4f7f\u7528 <code>qperf -t 60 &lt;server ip&gt; -ub -oo msg_size:1 -vu tcp_lat tcp_bw udp_lat udp_bw</code> \u6d4b\u8bd5  1 \u5b57\u8282\u5c0f\u5305\u4e0b tcp/udp \u7684\u5e26\u5bbd\u548c\u5ef6\u8fdf\uff0c\u5206\u522b\u6d4b\u8bd5\u4f18\u5316\u524d\uff0c\u4f18\u5316\u540e\u4ee5\u53ca\u5bbf\u4e3b\u673a\u7f51\u7edc\u7684\u6027\u80fd\uff1a</p> Type tcp_lat (us) udp_lat (us) tcp_bw (Mb/s) udp_bw(Mb/s) Kube-OVN Default 25.7 22.9 27.1 1.59 Kube-OVN Optimized 13.9 12.9 27.6 5.57 HOST Network 13.1 12.4 28.2 6.02"},{"location":"advance/performance-tuning/#overlay-underlay-calico","title":"Overlay\uff0c Underlay \u4ee5\u53ca Calico \u4e0d\u540c\u6a21\u5f0f\u6027\u80fd\u5bf9\u6bd4","text":"<p>\u4e0b\u9762\u6211\u4eec\u4f1a\u6bd4\u8f83\u4f18\u5316\u540e Kube-OVN \u5728\u4e0d\u540c\u5305\u5927\u5c0f\u4e0b\u7684 Overlay \u548c Underlay \u6027\u80fd\uff0c\u5e76\u548c Calico \u7684 <code>IPIP Always</code>, <code>IPIP never</code> \u4ee5\u53ca\u5bbf\u4e3b\u673a\u7f51\u7edc\u505a\u6bd4\u8f83\u3002</p> <p>Environment:</p> <ul> <li>Kubernetes: 1.22.0</li> <li>OS: CentOS 7</li> <li>Kube-OVN: 1.8.0</li> <li>CPU: AMD EPYC 7402P 24-Core Processor</li> <li>Network: Intel Corporation Ethernet Controller XXV710 for 25GbE SFP28</li> </ul> <p><code>qperf -t 60 &lt;server ip&gt; -ub -oo msg_size:1 -vu tcp_lat tcp_bw udp_lat udp_bw</code></p> Type tcp_lat (us) udp_lat (us) tcp_bw (Mb/s) udp_bw(Mb/s) Kube-OVN Overlay 15.2 14.6 23.6 2.65 Kube-OVN Underlay 14.3 13.8 24.2 3.46 Calico IPIP 21.4 20.2 23.6 1.18 Calico NoEncap 19.3 16.9 23.6 1.76 HOST Network 16.6 15.4 24.8 2.64 <p><code>qperf -t 60 &lt;server ip&gt; -ub -oo msg_size:1K -vu tcp_lat tcp_bw udp_lat udp_bw</code></p> Type tcp_lat (us) udp_lat (us) tcp_bw (Gb/s) udp_bw(Gb/s) Kube-OVN Overlay 16.5 15.8 10.2 2.77 Kube-OVN Underlay 15.9 14.5 9.6 3.22 Calico IPIP 22.5 21.5 1.45 1.14 Calico NoEncap 19.4 18.3 3.76 1.63 HOST Network 18.1 16.6 9.32 2.66 <p><code>qperf -t 60 &lt;server ip&gt; -ub -oo msg_size:4K -vu tcp_lat tcp_bw udp_lat udp_bw</code></p> Type tcp_lat (us) udp_lat (us) tcp_bw (Gb/s) udp_bw(Gb/s) Kube-OVN Overlay 34.7 41.6 16.0 9.23 Kube-OVN Underlay 32.6 44 15.1 6.71 Calico IPIP 44.8 52.9 2.94 3.26 Calico NoEncap 40 49.6 6.56 4.19 HOST Network 35.9 45.9 14.6 5.59 <p>\u5728\u90e8\u5206\u60c5\u51b5\u4e0b\u5bb9\u5668\u7f51\u7edc\u7684\u6027\u80fd\u4f1a\u4f18\u4e8e\u5bbf\u4e3b\u673a\u7f51\u7edc\uff0c\u8fd9\u662f\u4f18\u4e8e\u7ecf\u8fc7\u4f18\u5316\u540e\u5bb9\u5668\u7f51\u7edc\u8def\u5f84\u5b8c\u5168\u7ed5\u8fc7\u4e86 netfilter\uff0c \u800c\u5bbf\u4e3b\u673a\u7f51\u7edc\u7531\u4e8e <code>kube-proxy</code> \u7684\u5b58\u5728\u6240\u6709\u6570\u636e\u5305\u5747\u9700\u7ecf\u8fc7 netfilter\uff0c\u4f1a\u5bfc\u81f4\u5728\u4e00\u4e9b\u73af\u5883\u4e0b\u5bb9\u5668\u7f51\u7edc \u7684\u6d88\u8017\u76f8\u5bf9\u66f4\u5c0f\uff0c\u56e0\u6b64\u4f1a\u6709\u66f4\u597d\u7684\u6027\u80fd\u8868\u73b0\u3002</p>"},{"location":"advance/performance-tuning/#_3","title":"\u6570\u636e\u5e73\u9762\u6027\u80fd\u4f18\u5316\u65b9\u6cd5","text":"<p>\u8fd9\u91cc\u4ecb\u7ecd\u7684\u4f18\u5316\u65b9\u6cd5\u548c\u8f6f\u786c\u4ef6\u73af\u5883\u4ee5\u53ca\u6240\u9700\u8981\u7684\u529f\u80fd\u76f8\u5173\uff0c\u8bf7\u4ed4\u7ec6\u4e86\u89e3\u4f18\u5316\u7684\u524d\u63d0\u6761\u4ef6\u518d\u8fdb\u884c\u5c1d\u8bd5\u3002</p>"},{"location":"advance/performance-tuning/#cpu","title":"CPU \u6027\u80fd\u6a21\u5f0f\u8c03\u6574","text":"<p>\u90e8\u5206\u73af\u5883\u4e0b CPU \u8fd0\u884c\u5728\u8282\u80fd\u6a21\u5f0f\uff0c\u8be5\u6a21\u5f0f\u4e0b\u6027\u80fd\u8868\u73b0\u5c06\u4f1a\u4e0d\u7a33\u5b9a\uff0c\u5ef6\u8fdf\u4f1a\u51fa\u73b0\u660e\u663e\u589e\u52a0\uff0c\u5efa\u8bae\u4f7f\u7528 CPU \u7684\u6027\u80fd\u6a21\u5f0f\u83b7\u5f97\u66f4\u7a33\u5b9a\u7684\u6027\u80fd\u8868\u73b0\uff1a</p> <pre><code>cpupower frequency-set -g performance\n</code></pre>"},{"location":"advance/performance-tuning/#_4","title":"\u7f51\u5361\u786c\u4ef6\u961f\u5217\u8c03\u6574","text":"<p>\u5728\u6d41\u91cf\u589e\u5927\u7684\u60c5\u51b5\u4e0b\uff0c\u7f13\u51b2\u961f\u5217\u8fc7\u77ed\u53ef\u80fd\u5bfc\u81f4\u8f83\u9ad8\u7684\u4e22\u5305\u7387\u5bfc\u81f4\u6027\u80fd\u663e\u8457\u4e0b\u964d\uff0c\u9700\u8981\u8fdb\u884c\u8c03\u6574</p> <p>\u68c0\u67e5\u5f53\u524d\u7f51\u5361\u961f\u5217\u957f\u5ea6\uff1a</p> <pre><code># ethtool -g eno1\nRing parameters for eno1:\n Pre-set maximums:\n RX:             4096\nRX Mini:        0\nRX Jumbo:       0\nTX:             4096\nCurrent hardware settings:\n RX:             255\nRX Mini:        0\nRX Jumbo:       0\nTX:             255\n</code></pre> <p>\u589e\u52a0\u961f\u5217\u957f\u5ea6\u81f3\u6700\u5927\u503c\uff1a</p> <pre><code>ethtool -G eno1 rx 4096\nethtool -G eno1 tx 4096\n</code></pre>"},{"location":"advance/performance-tuning/#tuned","title":"\u4f7f\u7528 tuned \u4f18\u5316\u7cfb\u7edf\u53c2\u6570","text":"<p>tuned \u53ef\u4ee5\u4f7f\u7528\u4e00\u7cfb\u5217\u9884\u7f6e\u7684 profile \u6587\u4ef6\u4fdd\u5b58\u4e86\u9488\u5bf9\u7279\u5b9a\u573a\u666f\u7684\u4e00\u7cfb\u5217\u7cfb\u7edf\u4f18\u5316\u914d\u7f6e\u3002</p> <p>\u9488\u5bf9\u5ef6\u8fdf\u4f18\u5148\u573a\u666f\uff1a</p> <pre><code>tuned-adm profile network-latency\n</code></pre> <p>\u9488\u5bf9\u541e\u5410\u91cf\u4f18\u5148\u573a\u666f\uff1a</p> <pre><code>tuned-adm profile network-throughput\n</code></pre>"},{"location":"advance/performance-tuning/#_5","title":"\u4e2d\u65ad\u7ed1\u5b9a","text":"<p>\u6211\u4eec\u63a8\u8350\u7981\u7528 <code>irqbalance</code> \u5e76\u5c06\u7f51\u5361\u4e2d\u65ad\u548c\u7279\u5b9a CPU \u8fdb\u884c\u7ed1\u5b9a\uff0c\u6765\u907f\u514d\u5728\u591a\u4e2a CPU \u4e4b\u95f4\u5207\u6362\u5bfc\u81f4\u7684\u6027\u80fd\u6ce2\u52a8\u3002</p>"},{"location":"advance/performance-tuning/#ovn-lb","title":"\u5173\u95ed OVN LB","text":"<p>OVN \u7684 L2 LB \u5b9e\u73b0\u8fc7\u7a0b\u4e2d\u9700\u8981\u8c03\u7528\u5185\u6838\u7684 <code>conntrack</code> \u6a21\u5757\u5e76\u8fdb\u884c recirculate \u5bfc\u81f4\u5927\u91cf\u7684 CPU \u5f00\u9500\uff0c\u7ecf\u6d4b\u8bd5\u8be5\u529f\u80fd\u4f1a\u5e26\u6765 20% \u5de6\u53f3\u7684 CPU \u5f00\u9500\uff0c \u5728 Overlay \u7f51\u7edc\u6a21\u5f0f\u4e0b\u53ef\u4ee5\u4f7f\u7528 <code>kube-proxy</code> \u5b8c\u6210 Service \u8f6c\u53d1\u529f\u80fd\uff0c\u83b7\u5f97\u66f4\u597d\u7684 Pod-to-Pod \u6027\u80fd\u3002\u53ef\u4ee5\u5728 <code>kube-ovn-controller</code> \u4e2d\u5173\u95ed\u8be5\u529f\u80fd\uff1a</p> <pre><code>command:\n- /kube-ovn/start-controller.sh\nargs:\n...\n- --enable-lb=false\n...\n</code></pre> <p>Underlay \u6a21\u5f0f\u4e0b <code>kube-proxy</code> \u65e0\u6cd5\u4f7f\u7528 iptables \u6216 ipvs \u63a7\u5236\u5bb9\u5668\u7f51\u7edc\u6d41\u91cf\uff0c\u5982\u9700\u5173\u95ed LB \u529f\u80fd\u9700\u8981\u786e\u8ba4\u662f\u5426\u4e0d\u9700\u8981 Service \u529f\u80fd\u3002</p>"},{"location":"advance/performance-tuning/#fastpath","title":"\u5185\u6838 FastPath \u6a21\u5757","text":"<p>\u7531\u4e8e\u5bb9\u5668\u7f51\u7edc\u548c\u5bbf\u4e3b\u673a\u7f51\u7edc\u5728\u4e0d\u540c\u7684 network ns\uff0c\u6570\u636e\u5305\u5728\u8de8\u5bbf\u4e3b\u673a\u4f20\u8f93\u65f6\u4f1a\u591a\u6b21\u7ecf\u8fc7 netfilter \u6a21\u5757\uff0c\u4f1a\u5e26\u6765\u8fd1 20% \u7684 CPU \u5f00\u9500\u3002\u7531\u4e8e\u5927\u90e8\u5206\u60c5\u51b5\u4e0b \u5bb9\u5668\u7f51\u7edc\u5185\u5e94\u7528\u65e0\u987b\u4f7f\u7528 netfilter \u6a21\u5757\u7684\u529f\u80fd\uff0c<code>FastPath</code> \u6a21\u5757\u53ef\u4ee5\u7ed5\u8fc7 netfilter \u964d\u4f4e CPU \u5f00\u9500\u3002</p> <p>\u5982\u5bb9\u5668\u7f51\u7edc\u5185\u9700\u8981\u4f7f\u7528 netfilter \u63d0\u4f9b\u7684\u529f\u80fd\u5982 iptables\uff0cipvs\uff0cnftables \u7b49\uff0c\u8be5\u6a21\u5757\u4f1a\u4f7f\u76f8\u5173\u529f\u80fd\u5931\u6548\u3002</p> <p>\u7531\u4e8e\u5185\u6838\u6a21\u5757\u548c\u5185\u6838\u7248\u672c\u76f8\u5173\uff0c\u65e0\u6cd5\u63d0\u4f9b\u4e00\u4e2a\u5355\u4e00\u9002\u5e94\u6240\u6709\u5185\u6838\u7684\u5185\u6838\u6a21\u5757\u5236\u54c1\u3002\u6211\u4eec\u9884\u5148\u7f16\u8bd1\u4e86\u90e8\u5206\u5185\u6838\u7684 <code>FastPath</code> \u6a21\u5757\uff0c \u53ef\u4ee5\u524d\u5f80 tunning-package \u8fdb\u884c\u4e0b\u8f7d\u3002</p> <p>\u4e5f\u53ef\u4ee5\u624b\u52a8\u8fdb\u884c\u7f16\u8bd1\uff0c\u65b9\u6cd5\u53c2\u8003\u624b\u52a8\u7f16\u8bd1 FastPath \u6a21\u5757</p> <p>\u83b7\u5f97\u5185\u6838\u6a21\u5757\u540e\u53ef\u5728\u6bcf\u4e2a\u8282\u70b9\u4f7f\u7528 <code>insmod kube_ovn_fastpath.ko</code> \u52a0\u8f7d <code>FastPath</code> \u6a21\u5757\uff0c\u5e76\u4f7f\u7528 <code>dmesg</code> \u9a8c\u8bc1\u6a21\u5757\u52a0\u8f7d\u6210\u529f\uff1a</p> <pre><code># dmesg\n...\n[619631.323788] init_module,kube_ovn_fastpath_local_out\n[619631.323798] init_module,kube_ovn_fastpath_post_routing\n[619631.323800] init_module,kube_ovn_fastpath_pre_routing\n[619631.323801] init_module,kube_ovn_fastpath_local_in\n...\n</code></pre>"},{"location":"advance/performance-tuning/#ovs","title":"OVS \u5185\u6838\u6a21\u5757\u4f18\u5316","text":"<p>OVS \u7684 flow \u5904\u7406\u5305\u62ec\u54c8\u5e0c\u8ba1\u7b97\uff0c\u5339\u914d\u7b49\u64cd\u4f5c\u4f1a\u6d88\u8017\u5927\u7ea6 10% \u5de6\u53f3\u7684 CPU \u8d44\u6e90\u3002\u73b0\u4ee3 x86 CPU \u4e0a\u7684\u4e00\u4e9b\u6307\u4ee4\u96c6\u4f8b\u5982 <code>popcnt</code> \u548c <code>sse4.2</code> \u53ef\u4ee5 \u52a0\u901f\u76f8\u5173\u8ba1\u7b97\u8fc7\u7a0b\uff0c\u4f46\u5185\u6838\u9ed8\u8ba4\u7f16\u8bd1\u672a\u5f00\u542f\u76f8\u5173\u9009\u9879\u3002\u7ecf\u6d4b\u8bd5\u5728\u5f00\u542f\u76f8\u5e94\u6307\u4ee4\u96c6\u4f18\u5316\u540e\uff0cflow \u76f8\u5173\u64cd\u4f5c CPU \u6d88\u8017\u5c06\u4f1a\u964d\u81f3 5% \u5de6\u53f3\u3002</p> <p>\u548c <code>FastPath</code> \u6a21\u5757\u7684\u7f16\u8bd1\u7c7b\u4f3c\uff0c\u7531\u4e8e\u5185\u6838\u6a21\u5757\u548c\u5185\u6838\u7248\u672c\u76f8\u5173\uff0c\u65e0\u6cd5\u63d0\u4f9b\u4e00\u4e2a\u5355\u4e00\u9002\u5e94\u6240\u6709\u5185\u6838\u7684\u5185\u6838\u6a21\u5757\u5236\u54c1\u3002\u7528\u6237\u9700\u8981\u624b\u52a8\u7f16\u8bd1\u6216\u8005 \u524d\u5f80 tunning-package \u67e5\u770b\u662f\u5426\u6709\u5df2\u7f16\u8bd1\u597d\u7684\u5236\u54c1\u8fdb\u884c\u4e0b\u8f7d\u3002</p> <p>\u4f7f\u7528\u8be5\u5185\u6838\u6a21\u5757\u524d\u8bf7\u5148\u786e\u8ba4 CPU \u662f\u5426\u652f\u6301\u76f8\u5173\u6307\u4ee4\u96c6\uff1a</p> <pre><code>cat /proc/cpuinfo  | grep popcnt\ncat /proc/cpuinfo  | grep sse4_2\n</code></pre>"},{"location":"advance/performance-tuning/#centos","title":"CentOS \u4e0b\u7f16\u8bd1\u5b89\u88c5","text":"<p>\u5b89\u88c5\u76f8\u5173\u7f16\u8bd1\u4f9d\u8d56\u548c\u5185\u6838\u5934\u6587\u4ef6\uff1a</p> <pre><code>yum install -y gcc kernel-devel-$(uname -r) python3 autoconf automake libtool rpm-build openssl-devel\n</code></pre> <p>\u7f16\u8bd1 OVS \u5185\u6838\u6a21\u5757\u5e76\u751f\u6210\u5bf9\u5e94 RPM \u6587\u4ef6:</p> <pre><code>git clone -b branch-2.17 --depth=1 https://github.com/openvswitch/ovs.git\ncd ovs\ncurl -s  https://github.com/kubeovn/ovs/commit/2d2c83c26d4217446918f39d5cd5838e9ac27b32.patch |  git apply\n./boot.sh\n./configure --with-linux=/lib/modules/$(uname -r)/build CFLAGS=\"-g -O2 -mpopcnt -msse4.2\"\nmake rpm-fedora-kmod\ncd rpm/rpmbuild/RPMS/x86_64/\n</code></pre> <p>\u590d\u5236 RPM \u5230\u6bcf\u4e2a\u8282\u70b9\u5e76\u8fdb\u884c\u5b89\u88c5\uff1a</p> <pre><code>rpm -i openvswitch-kmod-2.15.2-1.el7.x86_64.rpm\n</code></pre> <p>\u82e5\u4e4b\u524d\u5df2\u7ecf\u542f\u52a8\u8fc7 Kube-OVN\uff0c\u65e7\u7248\u672c OVS \u6a21\u5757\u5df2\u52a0\u8f7d\u81f3\u5185\u6838\uff0c\u5efa\u8bae\u91cd\u542f\u673a\u5668\u91cd\u65b0\u52a0\u8f7d\u65b0\u7248\u5185\u6838\u6a21\u5757\u3002</p>"},{"location":"advance/performance-tuning/#ubuntu","title":"Ubuntu \u4e0b\u7f16\u8bd1\u5b89\u88c5","text":"<p>\u5b89\u88c5\u76f8\u5173\u7f16\u8bd1\u4f9d\u8d56\u548c\u5185\u6838\u5934\u6587\u4ef6\uff1a</p> <pre><code>apt install -y autoconf automake libtool gcc build-essential libssl-dev\n</code></pre> <p>\u7f16\u8bd1 OVS \u5185\u6838\u6a21\u5757\u5e76\u5b89\u88c5\uff1a</p> <pre><code>apt install -y autoconf automake libtool gcc build-essential libssl-dev\n\ngit clone -b branch-2.17 --depth=1 https://github.com/openvswitch/ovs.git\ncd ovs\ncurl -s  https://github.com/kubeovn/ovs/commit/2d2c83c26d4217446918f39d5cd5838e9ac27b32.patch |  git apply\n./boot.sh\n./configure --prefix=/usr/ --localstatedir=/var --enable-ssl --with-linux=/lib/modules/$(uname -r)/build\nmake -j `nproc`\nmake install\nmake modules_install\n\ncat &gt; /etc/depmod.d/openvswitch.conf &lt;&lt; EOF\noverride openvswitch * extra\noverride vport-* * extra\nEOF\n\ndepmod -a\ncp debian/openvswitch-switch.init /etc/init.d/openvswitch-switch\n/etc/init.d/openvswitch-switch force-reload-kmod\n</code></pre> <p>\u82e5\u4e4b\u524d\u5df2\u7ecf\u542f\u52a8\u8fc7 Kube-OVN\uff0c\u65e7\u7248\u672c OVS \u6a21\u5757\u5df2\u52a0\u8f7d\u81f3\u5185\u6838\uff0c\u5efa\u8bae\u91cd\u542f\u673a\u5668\u91cd\u65b0\u52a0\u8f7d\u65b0\u7248\u5185\u6838\u6a21\u5757\u3002</p>"},{"location":"advance/performance-tuning/#stt","title":"\u4f7f\u7528 STT \u7c7b\u578b\u96a7\u9053","text":"<p>\u5e38\u89c1\u7684\u96a7\u9053\u5c01\u88c5\u534f\u8bae\u4f8b\u5982 Geneve \u548c Vxlan \u4f7f\u7528 UDP \u534f\u8bae\u5bf9\u6570\u636e\u5305\u8fdb\u884c\u5c01\u88c5\uff0c\u5728\u5185\u6838\u4e2d\u6709\u826f\u597d\u7684\u652f\u6301\u3002\u4f46\u662f\u5f53\u4f7f\u7528 UDP \u5c01\u88c5 TCP \u6570\u636e\u5305\u65f6\uff0c \u73b0\u4ee3\u64cd\u4f5c\u7cfb\u7edf\u548c\u7f51\u5361\u9488\u5bf9 TCP \u534f\u8bae\u7684\u4f18\u5316\u548c offload \u529f\u80fd\u5c06\u65e0\u6cd5\u987a\u5229\u5de5\u4f5c\uff0c\u5bfc\u81f4 TCP \u7684\u541e\u5410\u91cf\u51fa\u73b0\u663e\u8457\u4e0b\u964d\u3002\u5728\u865a\u62df\u5316\u573a\u666f\u4e0b\u7531\u4e8e CPU \u7684\u9650\u5236\uff0c TCP \u5927\u5305\u7684\u541e\u5410\u91cf\u751a\u81f3\u53ef\u80fd\u53ea\u6709\u5bbf\u4e3b\u673a\u7f51\u7edc\u7684\u5341\u5206\u4e4b\u4e00\u3002</p> <p>STT \u63d0\u4f9b\u4e86\u4e00\u79cd\u521b\u65b0\u5f0f\u7684\u4f7f\u7528 TCP \u683c\u5f0f\u6570\u636e\u5305\u8fdb\u884c\u5c01\u88c5\u7684\u96a7\u9053\u534f\u8bae\uff0c\u8be5\u5c01\u88c5\u53ea\u662f\u6a21\u62df\u4e86 TCP \u534f\u8bae\u7684\u5934\u90e8\u683c\u5f0f\uff0c\u5e76\u6ca1\u6709\u771f\u6b63\u5efa\u7acb TCP \u8fde\u63a5\uff0c\u4f46\u662f\u53ef\u4ee5 \u5145\u5206\u5229\u7528\u73b0\u4ee3\u64cd\u4f5c\u7cfb\u7edf\u548c\u7f51\u5361\u7684 TCP \u4f18\u5316\u80fd\u529b\u3002\u5728\u6211\u4eec\u7684\u6d4b\u8bd5\u4e2d TCP \u5927\u5305\u7684\u541e\u5410\u91cf\u80fd\u6709\u6570\u500d\u7684\u63d0\u5347\uff0c\u8fbe\u5230\u63a5\u8fd1\u5bbf\u4e3b\u673a\u7f51\u7edc\u7684\u6027\u80fd\u6c34\u5e73\u3002</p> <p>STT \u96a7\u9053\u5e76\u6ca1\u6709\u9884\u5b89\u88c5\u5728\u5185\u6838\u5185\uff0c\u9700\u8981\u901a\u8fc7\u7f16\u8bd1 OVS \u5185\u6838\u6a21\u5757\u6765\u5b89\u88c5\uff0cOVS \u5185\u6838\u6a21\u5757\u7684\u7f16\u8bd1\u65b9\u6cd5\u53ef\u4ee5\u53c2\u8003\u4e0a\u4e00\u8282\u3002</p> <p>STT \u96a7\u9053\u5f00\u542f\uff1a</p> <pre><code>kubectl set env daemonset/ovs-ovn -n kube-system TUNNEL_TYPE=stt\n\nkubectl delete pod -n kube-system -lapp=ovs\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/security-group/","title":"SecurityGroup \u4f7f\u7528","text":"<p>Kube-OVN \u652f\u6301\u4e86\u5b89\u5168\u7ec4\u7684\u914d\u7f6e\uff0c\u914d\u7f6e\u5b89\u5168\u7ec4\u4f7f\u7528\u7684 CRD \u4e3a SecurityGroup\u3002</p>"},{"location":"advance/security-group/#_1","title":"\u5b89\u5168\u7ec4\u793a\u4f8b","text":"<pre><code>apiVersion: kubeovn.io/v1\nkind: SecurityGroup\nmetadata:\nname: sg-example\nspec:\nallowSameGroupTraffic: true\negressRules:\n- ipVersion: ipv4\npolicy: allow\npriority: 1\nprotocol: all\nremoteAddress: 10.16.0.13/16\nremoteType: address\ningressRules:\n- ipVersion: ipv4\npolicy: deny\npriority: 1\nprotocol: icmp\nremoteAddress: 10.16.0.14/16\nremoteType: address\n</code></pre> <p>\u5b89\u5168\u7ec4\u5404\u5b57\u6bb5\u7684\u5177\u4f53\u542b\u4e49\uff0c\u53ef\u4ee5\u53c2\u8003 Kube-OVN \u63a5\u53e3\u89c4\u8303\u3002</p> <p>Pod \u901a\u8fc7\u6dfb\u52a0 annotation \u6765\u7ed1\u5b9a\u5b89\u5168\u7ec4\uff0c\u4f7f\u7528\u7684 annotation \u6709\u4e24\u4e2a\uff1a</p> <pre><code>    ovn.kubernetes.io/port_security: \"true\"\novn.kubernetes.io/security_groups: sg-example\n</code></pre>"},{"location":"advance/security-group/#_2","title":"\u6ce8\u610f\u4e8b\u9879","text":"<ul> <li> <p>\u5b89\u5168\u7ec4\u6700\u540e\u662f\u901a\u8fc7\u8bbe\u7f6e ACL \u89c4\u5219\u6765\u9650\u5236\u8bbf\u95ee\u7684\uff0cOVN \u6587\u6863\u4e2d\u63d0\u5230\uff0c\u5982\u679c\u5339\u914d\u5230\u7684\u4e24\u4e2a ACL \u89c4\u5219\u62e5\u6709\u76f8\u540c\u7684\u4f18\u5148\u7ea7\uff0c\u5b9e\u9645\u8d77\u4f5c\u7528\u7684\u662f\u54ea\u4e2a ACL \u662f\u4e0d\u786e\u5b9a\u7684\u3002\u56e0\u6b64\u8bbe\u7f6e\u5b89\u5168\u7ec4\u89c4\u5219\u7684\u65f6\u5019\uff0c\u9700\u8981\u6ce8\u610f\u533a\u5206\u4f18\u5148\u7ea7\u3002</p> </li> <li> <p>\u5f53\u6dfb\u52a0\u5b89\u5168\u7ec4\u7684\u65f6\u5019\uff0c\u8981\u6e05\u695a\u7684\u77e5\u9053\u662f\u5728\u6dfb\u52a0\u4ec0\u4e48\u9650\u5236\u3002Kube-OVN \u4f5c\u4e3a CNI\uff0c\u521b\u5efa Pod \u540e\u4f1a\u8fdb\u884c Pod \u5230\u7f51\u5173\u7684\u8fde\u901a\u6027\u6d4b\u8bd5\uff0c\u5982\u679c\u8bbf\u95ee\u4e0d\u901a\u7f51\u5173\uff0c\u5c31\u4f1a\u5bfc\u81f4 Pod \u4e00\u76f4\u5904\u4e8e ContainerCreating \u72b6\u6001\uff0c\u65e0\u6cd5\u987a\u5229\u5207\u6362\u5230 Running \u72b6\u6001\u3002</p> </li> </ul>"},{"location":"advance/security-group/#_3","title":"\u5b9e\u9645\u6d4b\u8bd5","text":"<p>\u5229\u7528\u4ee5\u4e0b yaml \u521b\u5efa Pod\uff0c\u5728 annotation \u4e2d\u6307\u5b9a\u7ed1\u5b9a\u793a\u4f8b\u4e2d\u7684\u5b89\u5168\u7ec4\uff1a</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nlabels:\napp: static\nannotations:\novn.kubernetes.io/port_security: 'true'\novn.kubernetes.io/security_groups: 'sg-example'\nname: sg-test-pod\nnamespace: default\nspec:\nnodeName: kube-ovn-worker\ncontainers:\n- image: docker.io/library/nginx:alpine\nimagePullPolicy: IfNotPresent\nname: qatest\n</code></pre> <p>\u5b9e\u9645\u6d4b\u8bd5\u7ed3\u679c\u663e\u793a\u5982\u4e0b\uff1a</p> <pre><code># kubectl get pod -o wide\nNAME                   READY   STATUS              RESTARTS   AGE     IP           NODE                     NOMINATED NODE   READINESS GATES\nsg-test-pod            0/1     ContainerCreating   0          5h32m   &lt;none&gt;       kube-ovn-worker          &lt;none&gt;           &lt;none&gt;\ntest-99fff7f86-52h9r   1/1     Running             0          5h41m   10.16.0.14   kube-ovn-control-plane   &lt;none&gt;           &lt;none&gt;\ntest-99fff7f86-qcgjw   1/1     Running             0          5h43m   10.16.0.13   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;\n</code></pre> <p>\u6267\u884c kubectl describe pod \u67e5\u770b Pod \u7684\u4fe1\u606f\uff0c\u53ef\u4ee5\u770b\u5230\u62a5\u9519\u63d0\u793a\uff1a</p> <pre><code># kubectl describe pod sg-test-pod\nName:         sg-test-pod\nNamespace:    default\nPriority:     0\nNode:         kube-ovn-worker/172.18.0.2\nStart Time:   Tue, 28 Feb 2023 10:29:36 +0800\nLabels:       app=static\nAnnotations:  ovn.kubernetes.io/allocated: true\novn.kubernetes.io/cidr: 10.16.0.0/16\n              ovn.kubernetes.io/gateway: 10.16.0.1\n              ovn.kubernetes.io/ip_address: 10.16.0.15\n              ovn.kubernetes.io/logical_router: ovn-cluster\n              ovn.kubernetes.io/logical_switch: ovn-default\n              ovn.kubernetes.io/mac_address: 00:00:00:FA:17:97\n              ovn.kubernetes.io/pod_nic_type: veth-pair\n              ovn.kubernetes.io/port_security: true\novn.kubernetes.io/routed: true\novn.kubernetes.io/security_groups: sg-allow-reject\nStatus:       Pending\nIP:\nIPs:          &lt;none&gt;\n\u00b7\n\u00b7\n\u00b7\nEvents:\n  Type     Reason                  Age                    From     Message\n  ----     ------                  ----                   ----     -------\n  Warning  FailedCreatePodSandBox  5m3s (x70 over 4h59m)  kubelet  (combined from similar events): Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox \"40636e0c7f1ade5500fa958486163d74f2e2300051a71522a9afd7ba0538afb6\": plugin type=\"kube-ovn\" failed (add): RPC failed; request ip return 500 configure nic failed 10.16.0.15 network not ready after 200 ping 10.16.0.1\n</code></pre> <p>\u4fee\u6539\u5b89\u5168\u7ec4\u7684\u89c4\u5219\uff0c\u6dfb\u52a0\u5230\u7f51\u5173\u7684\u8bbf\u95ee\u89c4\u5219\uff0c\u53c2\u8003\u5982\u4e0b\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: SecurityGroup\nmetadata:\nname: sg-gw-both\nspec:\nallowSameGroupTraffic: true\negressRules:\n- ipVersion: ipv4\npolicy: allow\npriority: 2\nprotocol: all\nremoteAddress: 10.16.0.13/16\nremoteType: address\n- ipVersion: ipv4\npolicy: allow\npriority: 1\nprotocol: all\nremoteAddress: 10.16.0.1/16\nremoteType: address\ningressRules:\n- ipVersion: ipv4\npolicy: deny\npriority: 2\nprotocol: icmp\nremoteAddress: 10.16.0.14/16\nremoteType: address\n- ipVersion: ipv4\npolicy: allow\npriority: 1\nprotocol: icmp\nremoteAddress: 10.16.0.1/16\nremoteType: address\n</code></pre> <p>\u5206\u522b\u5728\u5165\u65b9\u5411\u548c\u51fa\u65b9\u5411\u89c4\u5219\u4e2d\uff0c\u6dfb\u52a0\u5141\u8bb8\u5230\u7f51\u5173\u7684\u8bbf\u95ee\u89c4\u5219\uff0c\u5e76\u4e14\u8bbe\u7f6e\u8be5\u89c4\u5219\u7684\u4f18\u5148\u7ea7\u6700\u9ad8\u3002</p> <p>\u5229\u7528\u4ee5\u4e0b yaml \u7ed1\u5b9a\u5b89\u5168\u7ec4\uff0c\u90e8\u7f72 Pod \u540e\uff0c\u786e\u8ba4 Pod \u53ef\u4ee5\u6b63\u5e38\u8fd0\u884c\uff1a</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nlabels:\napp: static\nannotations:\novn.kubernetes.io/port_security: 'true'\novn.kubernetes.io/security_groups: 'sg-gw-both'\nname: sg-gw-both\nnamespace: default\nspec:\nnodeName: kube-ovn-worker\ncontainers:\n- image: docker.io/library/nginx:alpine\nimagePullPolicy: IfNotPresent\nname: qatest\n</code></pre> <p>\u90e8\u7f72\u540e\u67e5\u770b Pod \u4fe1\u606f\uff1a</p> <pre><code># kubectl get pod -o wide\nNAME                   READY   STATUS              RESTARTS   AGE     IP           NODE                     NOMINATED NODE   READINESS GATES\nsg-test-pod            0/1     ContainerCreating   0          5h41m   &lt;none&gt;       kube-ovn-worker          &lt;none&gt;           &lt;none&gt;\nsg-gw-both             1/1     Running             0          5h37m   10.16.0.19   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;\n</code></pre> <p>\u56e0\u6b64\u5bf9\u4e8e\u5b89\u5168\u7ec4\u7684\u4f7f\u7528\uff0c\u8981\u7279\u522b\u660e\u786e\u6dfb\u52a0\u7684\u9650\u5236\u89c4\u5219\u7684\u4f5c\u7528\u3002\u5982\u679c\u5355\u7eaf\u662f\u9650\u5236\u6d41\u91cf\u8bbf\u95ee\uff0c\u53ef\u4ee5\u8003\u8651\u4f7f\u7528\u7f51\u7edc\u7b56\u7565\u5b9e\u73b0\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/vip/","title":"VIP \u9884\u7559\u8bbe\u7f6e","text":"<p>\u5728\u4e00\u4e9b\u573a\u666f\u4e0b\u6211\u4eec\u5e0c\u671b\u52a8\u6001\u7684\u9884\u7559\u4e00\u90e8\u5206 IP \u4f46\u662f\u5e76\u4e0d\u5206\u914d\u7ed9 Pod \u800c\u662f\u5206\u914d\u7ed9\u5176\u4ed6\u7684\u57fa\u7840\u8bbe\u65bd\u542f\u7528\uff0c\u4f8b\u5982\uff1a</p> <ul> <li>Kubernetes \u5d4c\u5957 Kubernetes \u7684\u573a\u666f\u4e2d\u4e0a\u5c42 Kubernetes \u4f7f\u7528 Underlay \u7f51\u7edc\u4f1a\u5360\u7528\u5e95\u5c42 Subnet \u53ef\u7528\u5730\u5740\u3002</li> <li>LB \u6216\u5176\u4ed6\u7f51\u7edc\u57fa\u7840\u8bbe\u65bd\u9700\u8981\u4f7f\u7528\u4e00\u4e2a Subnet \u5185\u7684 IP\uff0c\u4f46\u4e0d\u4f1a\u5355\u72ec\u8d77 Pod\u3002</li> </ul>"},{"location":"advance/vip/#vip_1","title":"\u521b\u5efa\u968f\u673a\u5730\u5740 VIP","text":"<p>\u5982\u679c\u53ea\u662f\u4e3a\u4e86\u9884\u7559\u82e5\u5e72 IP \u800c\u5bf9 IP \u5730\u5740\u672c\u8eab\u6ca1\u6709\u8981\u6c42\u53ef\u4ee5\u4f7f\u7528\u4e0b\u9762\u7684 yaml \u8fdb\u884c\u521b\u5efa\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Vip\nmetadata:\nname: vip-dynamic-01\nspec:\nsubnet: ovn-default\ntype: \"\"\n</code></pre> <ul> <li><code>subnet</code>: \u5c06\u4ece\u8be5 Subnet \u4e2d\u9884\u7559 IP\u3002</li> <li><code>type</code>: \u76ee\u524d\u652f\u6301\u4e24\u79cd\u7c7b\u578b\uff0c\u4e3a\u7a7a\u8868\u793a\u4ec5\u7528\u4e8e ipam ip \u5360\u4f4d\uff0c<code>switch_lb_vip</code> \u8868\u793a\u8be5 vip \u4ec5\u7528\u4e8e switch lb \u524d\u7aef vip \u548c\u540e\u7aef ip \u9700\u5904\u4e8e\u540c\u4e00\u5b50\u7f51\u3002</li> </ul> <p>\u521b\u5efa\u6210\u529f\u540e\u67e5\u8be2\u8be5 VIP\uff1a</p> <pre><code># kubectl get vip\nNAME             V4IP         PV4IP   MAC                 PMAC   V6IP   PV6IP   SUBNET        READY\nvip-dynamic-01   10.16.0.12           00:00:00:F0:DB:25                         ovn-default   true\n</code></pre> <p>\u53ef\u89c1\u8be5 VIP \u88ab\u5206\u914d\u4e86 <code>10.16.0.12</code> \u7684 IP \u5730\u5740\uff0c\u8be5\u5730\u5740\u53ef\u4ee5\u4e4b\u540e\u4f9b\u5176\u4ed6\u7f51\u7edc\u57fa\u7840\u8bbe\u65bd\u4f7f\u7528\u3002</p>"},{"location":"advance/vip/#vip_2","title":"\u521b\u5efa\u56fa\u5b9a\u5730\u5740 VIP","text":"<p>\u5982\u5bf9\u9884\u7559\u7684 VIP \u7684 IP \u5730\u5740\u6709\u9700\u6c42\u53ef\u4f7f\u7528\u4e0b\u9762\u7684 yaml \u8fdb\u884c\u56fa\u5b9a\u5206\u914d\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Vip\nmetadata:\nname: static-vip01\nspec:\nsubnet: ovn-default v4ip: \"10.16.0.121\"\n</code></pre> <ul> <li><code>subnet</code>: \u5c06\u4ece\u8be5 Subnet \u4e2d\u9884\u7559 IP\u3002</li> <li><code>v4ip</code>: \u56fa\u5b9a\u5206\u914d\u7684 IP \u5730\u5740\uff0c\u8be5\u5730\u5740\u9700\u5728 <code>subnet</code> \u7684 CIDR \u8303\u56f4\u5185\u3002</li> </ul> <p>\u521b\u5efa\u6210\u529f\u540e\u67e5\u8be2\u8be5 VIP\uff1a</p> <pre><code># kubectl get vip\nNAME             V4IP         PV4IP   MAC                 PMAC   V6IP   PV6IP   SUBNET        READY\nstatic-vip01   10.16.0.121           00:00:00:F0:DB:26                         ovn-default   true\n</code></pre> <p>\u53ef\u89c1\u8be5 VIP \u88ab\u5206\u914d\u4e86\u6240\u9884\u671f\u7684 IP \u5730\u5740\u3002</p>"},{"location":"advance/vip/#pod-vip-ip","title":"Pod \u4f7f\u7528 VIP \u6765\u56fa\u5b9a IP","text":"<p>\u8be5\u529f\u80fd\u4ece v1.12 \u5f00\u59cb\u652f\u6301\u3002</p> <p>\u53ef\u4ee5\u4f7f\u7528 annotation \u5c06\u67d0\u4e2a VIP \u5206\u914d\u7ed9\u4e00\u4e2a Pod\uff1a</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: static-ip\nannotations:\novn.kubernetes.io/vip: vip-dynamic-01 # \u6307\u5b9a vip\nnamespace: default\nspec:\ncontainers:\n- name: static-ip\nimage: docker.io/library/nginx:alpine\n</code></pre>"},{"location":"advance/vip/#statefulset-kubevirt-vm-vip","title":"StatefulSet \u548c Kubevirt VM \u4fdd\u7559 VIP","text":"<p>\u9488\u5bf9 <code>StatefulSet</code> \u548c <code>VM</code> \u7684\u7279\u6b8a\u6027\uff0c\u5728\u4ed6\u4eec\u7684 Pod \u9500\u6bc1\u518d\u62c9\u8d77\u8d77\u540e\u4f1a\u91cd\u65b0\u4f7f\u7528\u4e4b\u524d\u8bbe\u7f6e\u7684 VIP\u3002</p> <p>VM \u4fdd\u7559 VIP \u9700\u8981\u786e\u4fdd <code>kube-ovn-controller</code> \u7684 <code>keep-vm-ip</code> \u53c2\u6570\u4e3a <code>true</code>\u3002\u8bf7\u53c2\u8003 Kubevirt VM \u56fa\u5b9a\u5730\u5740\u5f00\u542f\u8bbe\u7f6e</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/vpc-dns/","title":"\u81ea\u5b9a\u4e49 VPC DNS","text":"<p>\u7531\u4e8e\u7528\u6237\u81ea\u5b9a\u4e49 VPC \u548c \u9ed8\u8ba4 VPC \u7f51\u7edc\u76f8\u4e92\u9694\u79bb\uff0c\u81ea\u5b9a VPC \u5185\u65e0\u6cd5\u8bbf\u95ee\u5230\u90e8\u7f72\u5728\u9ed8\u8ba4 VPC \u5185\u7684 coredns\u3002 \u5982\u679c\u7528\u6237\u5e0c\u671b\u5728\u81ea\u5b9a\u4e49 VPC \u5185\u4f7f\u7528 Kubernetes \u63d0\u4f9b\u7684\u96c6\u7fa4\u5185\u57df\u540d\u89e3\u6790\u80fd\u529b\uff0c\u53ef\u4ee5\u53c2\u8003\u672c\u6587\u6863\uff0c\u5229\u7528 <code>vpc-dns</code> CRD \u6765\u5b9e\u73b0\u3002</p> <p>\u8be5 CRD \u6700\u7ec8\u4f1a\u90e8\u7f72\u4e00\u4e2a coredns\uff0c\u8be5 Pod \u6709\u4e24\u4e2a\u7f51\u5361\uff0c\u4e00\u4e2a\u7f51\u5361\u5728\u7528\u6237\u81ea\u5b9a\u4e49 VPC\uff0c\u53e6\u4e00\u4e2a\u7f51\u5361\u5728\u9ed8\u8ba4 VPC \u4ece\u800c\u5b9e\u73b0\u7f51\u7edc\u4e92\u901a\uff0c\u540c\u65f6\u901a\u8fc7\u81ea\u5b9a\u4e49 VPC \u5185\u90e8\u8d1f\u8f7d\u5747\u8861\u63d0\u4f9b\u81ea\u5b9a\u4e49 VPC \u5185\u7684\u4e00\u4e2a\u5185\u90e8\u8d1f\u8f7d\u5747\u8861\u3002</p>"},{"location":"advance/vpc-dns/#vpc-dns_1","title":"\u90e8\u7f72 vpc-dns \u6240\u4f9d\u8d56\u7684\u8d44\u6e90","text":"<pre><code>apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\nlabels:\nkubernetes.io/bootstrapping: rbac-defaults\nname: system:vpc-dns\nrules:\n- apiGroups:\n- \"\"\nresources:\n- endpoints\n- services\n- pods\n- namespaces\nverbs:\n- list\n- watch\n- apiGroups:\n- discovery.k8s.io\nresources:\n- endpointslices\nverbs:\n- list\n- watch\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\nannotations:\nrbac.authorization.kubernetes.io/autoupdate: \"true\"\nlabels:\nkubernetes.io/bootstrapping: rbac-defaults\nname: vpc-dns\nroleRef:\napiGroup: rbac.authorization.k8s.io\nkind: ClusterRole\nname: system:vpc-dns\nsubjects:\n- kind: ServiceAccount\nname: vpc-dns\nnamespace: kube-system\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\nname: vpc-dns\nnamespace: kube-system\n---\napiVersion: v1\nkind: ConfigMap\nmetadata:\nname: vpc-dns-corefile\nnamespace: kube-system\ndata:\nCorefile: |\n.:53 {\nerrors\nhealth {\nlameduck 5s\n}\nready\nkubernetes cluster.local in-addr.arpa ip6.arpa {\npods insecure\nfallthrough in-addr.arpa ip6.arpa\n}\nprometheus :9153\nforward . /etc/resolv.conf {\nprefer_udp\n}\ncache 30\nloop\nreload\nloadbalance\n}\n</code></pre> <p>\u9664\u4e86\u4ee5\u4e0a\u8d44\u6e90\uff0c\u8be5\u529f\u80fd\u8fd8\u4f9d\u8d56 nat-gw-pod \u955c\u50cf\u8fdb\u884c\u8def\u7531\u914d\u7f6e\u3002</p>"},{"location":"advance/vpc-dns/#_1","title":"\u914d\u7f6e\u9644\u52a0\u7f51\u5361","text":"<pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: ovn-nad\nnamespace: default\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.0\",\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"ovn-nad.default.ovn\"\n}'\n</code></pre>"},{"location":"advance/vpc-dns/#ovn-default-provider","title":"\u4fee\u6539 ovn-default \u5b50\u7f51\u7684 provider","text":"<p>\u4fee\u6539 ovn-default \u7684 provider\uff0c\u4e3a\u4e0a\u9762 nad \u914d\u7f6e\u7684 provider <code>ovn-nad.default.ovn</code></p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: ovn-default\nspec:\ncidrBlock: 10.16.0.0/16\ndefault: true\ndisableGatewayCheck: false\ndisableInterConnection: false\nenableDHCP: false\nenableIPv6RA: false\nexcludeIps:\n- 10.16.0.1\ngateway: 10.16.0.1\ngatewayType: distributed\nlogicalGateway: false\nnatOutgoing: true\nprivate: false\nprotocol: IPv4\nprovider: ovn-nad.default.ovn # \u53ea\u9700\u4fee\u6539\u8be5\u5b57\u6bb5\nvpc: ovn-cluster\n</code></pre>"},{"location":"advance/vpc-dns/#vpc-dns-configmap","title":"\u914d\u7f6e vpc-dns \u7684 Configmap","text":"<p>\u5728 kube-system \u547d\u540d\u7a7a\u95f4\u4e0b\u521b\u5efa configmap\uff0c\u914d\u7f6e vpc-dns \u4f7f\u7528\u53c2\u6570\uff0c\u7528\u4e8e\u540e\u9762\u542f\u52a8 vpc-dns \u529f\u80fd\uff1a</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: vpc-dns-config\nnamespace: kube-system\ndata:\ncoredns-vip: 10.96.0.3\nenable-vpc-dns: \"true\"\nnad-name: ovn-nad\nnad-provider: ovn-nad.default.ovn\n</code></pre> <ul> <li><code>enable-vpc-dns</code>\uff1a\u662f\u5426\u542f\u7528\u529f\u80fd\uff0c\u9ed8\u8ba4 <code>true</code>\u3002</li> <li><code>coredns-image</code>\uff1adns \u90e8\u7f72\u955c\u50cf\u3002\u9ed8\u8ba4\u4e3a\u96c6\u7fa4 coredns \u90e8\u7f72\u7248\u672c\u3002</li> <li><code>coredns-vip</code>\uff1a\u4e3a coredns \u63d0\u4f9b lb \u670d\u52a1\u7684 vip\u3002</li> <li><code>coredns-template</code>\uff1acoredns \u90e8\u7f72\u6a21\u677f\u6240\u5728\u7684 URL\u3002\u9ed8\u8ba4\u83b7\u53d6\u5f53\u524d\u7248\u672c ovn \u76ee\u5f55\u4e0b <code>coredns-template.yaml</code> \u9ed8\u8ba4\u4e3a <code>https://raw.githubusercontent.com/kubeovn/kube-ovn/\u5f53\u524d\u7248\u672c/yamls/coredns-template.yaml</code> \u3002</li> <li><code>nad-name</code>\uff1a\u914d\u7f6e\u7684 <code>network-attachment-definitions</code> \u8d44\u6e90\u540d\u79f0\u3002</li> <li><code>nad-provider</code>\uff1a\u4f7f\u7528\u7684 provider \u540d\u79f0\u3002</li> <li><code>k8s-service-host</code>\uff1a\u7528\u4e8e coredns \u8bbf\u95ee k8s apiserver \u670d\u52a1\u7684 ip\uff0c\u9ed8\u8ba4\u4e3a\u96c6\u7fa4\u5185 apiserver \u5730\u5740\u3002</li> <li><code>k8s-service-port</code>\uff1a\u7528\u4e8e coredns \u8bbf\u95ee k8s apiserver \u670d\u52a1\u7684 port\uff0c\u9ed8\u8ba4\u4e3a\u96c6\u7fa4\u5185 apiserver \u7aef\u53e3\u3002</li> </ul>"},{"location":"advance/vpc-dns/#vpc-dns_2","title":"\u90e8\u7f72 vpc-dns","text":"<p>\u914d\u7f6e vpc-dns yaml\uff1a</p> <pre><code>kind: VpcDns\napiVersion: kubeovn.io/v1\nmetadata:\nname: test-cjh1\nspec:\nvpc: cjh-vpc-1\nsubnet: cjh-subnet-1\n</code></pre> <ul> <li><code>vpc</code> \uff1a \u7528\u4e8e\u90e8\u7f72 dns \u7ec4\u4ef6\u7684 vpc \u540d\u79f0\u3002</li> <li><code>subnet</code>\uff1a\u7528\u4e8e\u90e8\u7f72 dns \u7ec4\u4ef6\u7684\u5b50\u540d\u79f0\u3002</li> </ul> <p>\u67e5\u770b\u90e8\u7f72\u8d44\u6e90\u7684\u4fe1\u606f\uff1a</p> <pre><code># kubectl get vpc-dns\nNAME        ACTIVE   VPC         SUBNET   \ntest-cjh1   false    cjh-vpc-1   cjh-subnet-1   \ntest-cjh2   true     cjh-vpc-1   cjh-subnet-2 </code></pre> <p><code>ACTIVE</code> : <code>true</code> \u90e8\u7f72\u4e86\u81ea\u5b9a\u4e49 dns \u7ec4\u4ef6\uff0c<code>false</code> \u65e0\u90e8\u7f72\u3002</p> <ul> <li>\u9650\u5236\uff1a\u4e00\u4e2a VPC \u4e0b\u53ea\u4f1a\u90e8\u7f72\u4e00\u4e2a\u81ea\u5b9a\u4e49 dns \u7ec4\u4ef6;</li> <li>\u5f53\u4e00\u4e2a VPC \u4e0b\u914d\u7f6e\u591a\u4e2a vpc-dns \u8d44\u6e90\uff08\u5373\u540c\u4e00\u4e2a VPC \u4e0d\u540c\u7684 subnet\uff09\uff0c\u53ea\u6709\u4e00\u4e2a vpc-dns \u8d44\u6e90\u72b6\u6001 <code>true</code>\uff0c\u5176\u4ed6\u4e3a <code>fasle</code>;</li> <li>\u5f53 <code>true</code> \u7684 vpc-dns \u88ab\u5220\u9664\u6389\uff0c\u4f1a\u83b7\u53d6\u5176\u4ed6 <code>false</code> \u7684 vpc-dns \u8fdb\u884c\u90e8\u7f72\u3002</li> </ul>"},{"location":"advance/vpc-dns/#_2","title":"\u9a8c\u8bc1\u90e8\u7f72\u7ed3\u679c","text":"<p>\u67e5\u770b vpc-dns Pod \u72b6\u6001\uff0c\u4f7f\u7528 label <code>app=vpc-dns</code>\uff0c\u53ef\u4ee5\u67e5\u770b\u6240\u6709 vpc-dns pod \u72b6\u6001\uff1a</p> <pre><code># kubectl -n kube-system get pods -l app=vpc-dns\nNAME                                 READY   STATUS    RESTARTS   AGE\nvpc-dns-test-cjh1-7b878d96b4-g5979   1/1     Running   0          28s\nvpc-dns-test-cjh1-7b878d96b4-ltmf9   1/1     Running   0          28s\n</code></pre> <p>\u67e5\u770b slr \u72b6\u6001\u4fe1\u606f\uff1a</p> <pre><code># kubectl -n kube-system get slr\nNAME                VIP         PORT(S)                  SERVICE                             AGE\nvpc-dns-test-cjh1   10.96.0.3   53/UDP,53/TCP,9153/TCP   kube-system/slr-vpc-dns-test-cjh1   113s\n</code></pre> <p>\u8fdb\u5165\u8be5 VPC \u4e0b\u7684 Pod\uff0c\u6d4b\u8bd5 dns \u89e3\u6790:</p> <pre><code>nslookup kubernetes.default.svc.cluster.local 10.96.0.3\n</code></pre> <p>\u8be5 VPC \u4e0b\u7684 switch lb rule \u6240\u5728\u7684\u5b50\u7f51\u4ee5\u53ca\u540c\u4e00 VPC \u4e0b\u7684\u5176\u4ed6\u5b50\u7f51\u4e0b\u7684 pod \u90fd\u53ef\u4ee5\u89e3\u6790\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/vpc-internal-lb/","title":"\u81ea\u5b9a\u4e49 VPC \u5185\u90e8\u8d1f\u8f7d\u5747\u8861","text":"<p>Kubernetes \u63d0\u4f9b\u7684 Service \u53ef\u4ee5\u7528\u4f5c\u96c6\u7fa4\u5185\u7684\u8d1f\u8f7d\u5747\u8861\uff0c \u4f46\u662f\u5728\u81ea\u5b9a\u4e49 VPC \u6a21\u5f0f\u4e0b\uff0c \u4f7f\u7528 Service \u4f5c\u4e3a\u5185\u90e8\u8d1f\u8f7d\u5747\u8861\u5b58\u5728\u5982\u4e0b\u51e0\u4e2a\u95ee\u9898\uff1a</p> <ol> <li>Service IP \u8303\u56f4\u4e3a\u96c6\u7fa4\u8d44\u6e90\uff0c\u6240\u6709\u81ea\u5b9a\u4e49 VPC \u5171\u4eab\uff0c\u65e0\u6cd5\u91cd\u53e0\u3002</li> <li>\u7528\u6237\u65e0\u6cd5\u6309\u7167\u81ea\u5df1\u610f\u613f\u8bbe\u7f6e\u5185\u90e8\u8d1f\u8f7d\u5747\u8861\u7684 IP \u5730\u5740\u3002</li> </ol> <p>\u4e3a\u4e86\u89e3\u51b3\u4e0a\u8ff0\u95ee\u9898\uff0cKube-OVN \u5728 1.11 \u5f15\u5165 <code>SwitchLBRule</code> CRD\uff0c\u7528\u6237\u53ef\u4ee5\u8bbe\u7f6e\u81ea\u5b9a\u4e49 VPC \u5185\u7684\u5185\u90e8\u8d1f\u8f7d\u5747\u8861\u89c4\u5219\u3002</p> <p><code>SwitchLBRule</code> \u652f\u6301\u4ee5\u4e0b\u4e24\u79cd\u65b9\u5f0f\u8bbe\u7f6e\u81ea\u5b9a\u4e49 VPC \u5185\u7684\u5185\u90e8\u8d1f\u8f7d\u5747\u8861\u89c4\u5219\u3002</p>"},{"location":"advance/vpc-internal-lb/#selector","title":"<code>Selector</code> \u81ea\u52a8\u751f\u6210\u8d1f\u8f7d\u5747\u8861\u89c4\u5219","text":"<p>\u901a\u8fc7 <code>selector</code> \u53ef\u4ee5\u901a\u8fc7 <code>label</code> \u81ea\u52a8\u5173\u8054 <code>pod</code> \u914d\u7f6e\u751f\u6210\u8d1f\u8f7d\u5747\u8861\u89c4\u5219\u3002</p> <p><code>SwitchLBRule</code> \u6837\u4f8b\u5982\u4e0b\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: SwitchLBRule\nmetadata:\nname:  cjh-slr-nginx\nspec:\nvip: 1.1.1.1\nsessionAffinity: ClientIP\nnamespace: default\nselector:\n- app:nginx\nports:\n- name: dns\nport: 8888\ntargetPort: 80\nprotocol: TCP\n</code></pre> <ul> <li> <p><code>selector</code>, <code>sessionAffinity</code> \u548c <code>port</code> \u4f7f\u7528\u65b9\u5f0f\u540c Kubernetes Service\u3002</p> </li> <li> <p><code>vip</code>\uff1a\u81ea\u5b9a\u4e49\u8d1f\u8f7d\u5747\u8861\u7684 IP \u5730\u5740\u3002</p> </li> <li> <p><code>namespace</code>\uff1a<code>selector</code> \u6240\u9009\u62e9 Pod \u6240\u5728\u547d\u540d\u7a7a\u95f4\u3002</p> <p>Kube-OVN \u4f1a\u6839\u636e <code>SwitchLBRule</code> \u5b9a\u4e49\u9009\u62e9\u7684 Pod \u5f97\u51fa Pod \u6240\u5728 VPC \u5e76\u8bbe\u7f6e\u5bf9\u5e94\u7684 L2 LB\u3002</p> </li> </ul>"},{"location":"advance/vpc-internal-lb/#endpoints","title":"<code>Endpoints</code> \u81ea\u5b9a\u4e49\u8d1f\u8f7d\u5747\u8861\u89c4\u5219","text":"<p>\u901a\u8fc7 <code>endpoints</code> \u53ef\u4ee5\u81ea\u5b9a\u4e49\u8d1f\u8f7d\u5747\u8861\u89c4\u5219\uff0c\u7528\u4ee5\u652f\u6301\u65e0\u6cd5\u901a\u8fc7 <code>selector</code> \u81ea\u52a8\u751f\u6210\u8d1f\u8f7d\u5747\u8861\u89c4\u5219\u7684\u573a\u666f\uff0c\u6bd4\u5982\u8d1f\u8f7d\u5747\u8861\u540e\u7aef\u662f <code>kubevirt</code> \u521b\u5efa\u7684 <code>vm</code> \u3002</p> <p><code>SwitchLBRule</code> \u6837\u4f8b\u5982\u4e0b\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: SwitchLBRule\nmetadata:\nname:  cjh-slr-nginx\nspec:\nvip: 1.1.1.1\nsessionAffinity: ClientIP\nnamespace: default\nendpoints:\n- 192.168.0.101\n- 192.168.0.102\n- 192.168.0.103\nports:\n- name: dns\nport: 8888\ntargetPort: 80\nprotocol: TCP\n</code></pre> <ul> <li> <p><code>sessionAffinity</code> \u548c <code>port</code> \u4f7f\u7528\u65b9\u5f0f\u540c Kubernetes Service\u3002</p> </li> <li> <p><code>vip</code>\uff1a\u81ea\u5b9a\u4e49\u8d1f\u8f7d\u5747\u8861\u7684 IP \u5730\u5740\u3002</p> </li> <li> <p><code>namespace</code>\uff1a<code>selector</code> \u6240\u9009\u62e9 Pod \u6240\u5728\u547d\u540d\u7a7a\u95f4\u3002</p> </li> <li> <p><code>endpoints</code>\uff1a\u8d1f\u8f7d\u5747\u8861\u540e\u7aef IP \u5217\u8868\u3002</p> </li> </ul> <p>\u6ce8\uff1a\u5982\u679c\u540c\u65f6\u914d\u7f6e\u4e86 <code>selector</code> \u548c <code>endpoints</code>,\u4f1a\u81ea\u52a8\u5ffd\u7565<code>selector</code>\u914d\u7f6e\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/vpc-peering/","title":"VPC \u4e92\u8054","text":"<p>VPC \u4e92\u8054\u63d0\u4f9b\u4e86\u4e00\u79cd\u5c06\u4e24\u4e2a VPC \u7f51\u7edc\u901a\u8fc7\u903b\u8f91\u8def\u7531\u6253\u901a\u7684\u673a\u5236\uff0c\u4ece\u800c\u4f7f\u4e24\u4e2a VPC \u5185\u7684\u5de5\u4f5c\u8d1f\u8f7d\u53ef\u4ee5\u50cf\u5728\u540c\u4e00\u4e2a\u79c1\u6709\u7f51\u7edc\u4e00\u6837\uff0c \u901a\u8fc7\u79c1\u6709\u5730\u5740\u76f8\u4e92\u8bbf\u95ee\uff0c\u65e0\u9700\u901a\u8fc7\u5916\u90e8\u7f51\u5173\u8fdb\u884c NAT \u8f6c\u53d1\u3002</p>"},{"location":"advance/vpc-peering/#_1","title":"\u524d\u63d0\u6761\u4ef6","text":"<ol> <li>\u8be5\u529f\u80fd\u53ea\u9002\u7528\u4e8e\u7528\u6237\u81ea\u5b9a\u4e49 VPC\u3002</li> <li>\u4e3a\u4e86\u907f\u514d\u8def\u7531\u91cd\u53e0\u4e24\u4e2a VPC \u5185\u7684\u5b50\u7f51 CIDR \u4e0d\u80fd\u91cd\u53e0\u3002</li> <li>\u76ee\u524d\u53ea\u652f\u6301\u4e24\u4e2a VPC \u7684\u4e92\u8054\uff0c\u66f4\u591a\u7ec4 VPC \u4e4b\u95f4\u7684\u4e92\u8054\u6682\u4e0d\u652f\u6301\u3002</li> </ol>"},{"location":"advance/vpc-peering/#_2","title":"\u4f7f\u7528\u65b9\u5f0f","text":"<p>\u9996\u5148\u521b\u5efa\u4e24\u4e2a\u4e0d\u4e92\u8054\u7684 VPC\uff0c\u6bcf\u4e2a VPC \u4e0b\u5404\u6709\u4e00\u4e2a Subnet\uff0cSubnet \u7684 CIDR \u4e92\u4e0d\u91cd\u53e0\u3002</p> <pre><code>kind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: vpc-1\nspec: {}\n---\nkind: Subnet\napiVersion: kubeovn.io/v1\nmetadata:\nname: net1\nspec:\nvpc: vpc-1\ncidrBlock: 10.0.0.0/16\n---\nkind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: vpc-2\nspec: {}\n---\nkind: Subnet\napiVersion: kubeovn.io/v1\nmetadata:\nname: net2\nspec:\nvpc: vpc-2\ncidrBlock: 172.31.0.0/16\n</code></pre> <p>\u5728\u6bcf\u4e2a VPC \u5185\u5206\u522b\u589e\u52a0 <code>vpcPeerings</code> \u548c\u5bf9\u5e94\u7684\u9759\u6001\u8def\u7531\uff1a</p> <pre><code>kind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: vpc-1\nspec: vpcPeerings:\n- remoteVpc: vpc-2\nlocalConnectIP: 169.254.0.1/30\nstaticRoutes:\n- cidr: 172.31.0.0/16\nnextHopIP: 169.254.0.2\npolicy: policyDst\n---\nkind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: vpc-2\nspec:\nvpcPeerings:\n- remoteVpc: vpc-1\nlocalConnectIP: 169.254.0.2/30\nstaticRoutes:\n- cidr: 10.0.0.0/16\nnextHopIP: 169.254.0.1\npolicy: policyDst\n</code></pre> <ul> <li><code>remoteVpc</code>: \u4e92\u8054\u7684\u53e6\u4e00\u4e2a VPC \u7684\u540d\u5b57\u3002</li> <li><code>localConnectIP</code>: \u4f5c\u4e3a\u4e92\u8054\u7aef\u70b9\u7684 IP \u5730\u5740\u548c CIDR\uff0c\u6ce8\u610f\u4e24\u7aef IP \u5e94\u5c5e\u4e8e\u540c\u4e00 CIDR\uff0c\u4e14\u4e0d\u80fd\u548c\u5df2\u6709\u5b50\u7f51\u51b2\u7a81\u3002</li> <li><code>cidr</code>\uff1a\u53e6\u4e00\u7aef Subnet \u7684 CIDR\u3002</li> <li><code>nextHopIP</code>\uff1a\u4e92\u8054 VPC \u53e6\u4e00\u7aef\u7684 <code>localConnectIP</code>\u3002</li> </ul> <p>\u5206\u522b\u5728\u4e24\u4e2a Subnet \u4e0b\u521b\u5efa Pod</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nannotations:\novn.kubernetes.io/logical_switch: net1\nname: vpc-1-pod\nspec:\ncontainers:\n- name: vpc-1-pod\nimage: docker.io/library/nginx:alpine\n---\napiVersion: v1\nkind: Pod\nmetadata:\nannotations:\novn.kubernetes.io/logical_switch: net2\nname: vpc-2-pod\nspec:\ncontainers:\n- name: vpc-2-pod\nimage: docker.io/library/nginx:alpine\n</code></pre> <p>\u6d4b\u8bd5\u7f51\u7edc\u8fde\u901a\u6027</p> <pre><code># kubectl exec -it vpc-1-pod -- ping $(kubectl get pod vpc-2-pod -o jsonpath='{.status.podIP}')\nPING 172.31.0.2 (172.31.0.2): 56 data bytes\n64 bytes from 172.31.0.2: seq=0 ttl=62 time=0.655 ms\n64 bytes from 172.31.0.2: seq=1 ttl=62 time=0.086 ms\n64 bytes from 172.31.0.2: seq=2 ttl=62 time=0.098 ms\n^C\n--- 172.31.0.2 ping statistics ---\n3 packets transmitted, 3 packets received, 0% packet loss\nround-trip min/avg/max = 0.086/0.279/0.655 ms\n# kubectl exec -it vpc-2-pod -- ping $(kubectl get pod vpc-1-pod -o jsonpath='{.status.podIP}')\nPING 10.0.0.2 (10.0.0.2): 56 data bytes\n64 bytes from 10.0.0.2: seq=0 ttl=62 time=0.594 ms\n64 bytes from 10.0.0.2: seq=1 ttl=62 time=0.093 ms\n64 bytes from 10.0.0.2: seq=2 ttl=62 time=0.088 ms\n^C\n--- 10.0.0.2 ping statistics ---\n3 packets transmitted, 3 packets received, 0% packet loss\nround-trip min/avg/max = 0.088/0.258/0.594 ms\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/windows/","title":"Windows \u652f\u6301","text":"<p>Kube-OVN \u652f\u6301\u5305\u542b Windows \u7cfb\u7edf\u8282\u70b9\u7684 Kubernetes \u96c6\u7fa4\u7f51\u7edc\uff0c\u53ef\u4ee5\u5c06 Windows \u5bb9\u5668\u7684\u7f51\u7edc\u7edf\u4e00\u63a5\u5165\u8fdb\u884c\u7ba1\u7406\u3002</p>"},{"location":"advance/windows/#_1","title":"\u524d\u63d0\u6761\u4ef6","text":"<ul> <li>\u53c2\u8003 Adding Windows nodes \u589e\u52a0 Windows \u8282\u70b9\u3002</li> <li>Windows \u8282\u70b9\u5fc5\u987b\u5b89\u88c5 KB4489899 \u8865\u4e01\u4ee5\u4f7f Overlay/VXLAN \u7f51\u7edc\u6b63\u5e38\u5de5\u4f5c\uff0c\u5efa\u8bae\u66f4\u65b0\u7cfb\u7edf\u81f3\u6700\u65b0\u7248\u672c\u3002</li> <li>Windows \u8282\u70b9\u5fc5\u987b\u5b89\u88c5 Hyper-V \u53ca\u7ba1\u7406\u5de5\u5177\u3002</li> <li>\u7531\u4e8e Windows \u9650\u5236\u96a7\u9053\u5c01\u88c5\u53ea\u80fd\u4f7f\u7528 Vxlan \u6a21\u5f0f\u3002</li> <li>\u6682\u4e0d\u652f\u6301 SSL\uff0cIPv6\uff0c\u53cc\u6808\uff0cQoS \u529f\u80fd\u3002</li> <li>\u6682\u4e0d\u652f\u6301\u52a8\u6001\u5b50\u7f51\uff0c\u52a8\u6001\u96a7\u9053\u63a5\u53e3\u529f\u80fd\uff0c\u9700\u5728\u5b89\u88c5 Windows \u8282\u70b9\u524d\u5b8c\u6210\u5b50\u7f51\u521b\u5efa\uff0c\u5e76\u56fa\u5b9a\u7f51\u7edc\u63a5\u53e3\u3002</li> <li>\u4e0d\u652f\u6301\u591a\u4e2a <code>ProviderNetwork</code>\uff0c\u4e14\u65e0\u6cd5\u52a8\u6001\u8c03\u6574\u6865\u63a5\u63a5\u53e3\u914d\u7f6e\u3002</li> </ul>"},{"location":"advance/windows/#ovs","title":"\u5b89\u88c5 OVS","text":"<p>\u7531\u4e8e\u4e0a\u6e38 OVN \u548c OVS \u5bf9 Windows \u5bb9\u5668\u652f\u6301\u5b58\u5728\u4e00\u4e9b\u95ee\u9898\uff0c\u9700\u8981\u4f7f\u7528 Kube-OVN \u63d0\u4f9b\u7684\u7ecf\u8fc7\u4fee\u6539\u7684\u5b89\u88c5\u5305\u8fdb\u884c\u5b89\u88c5\u3002</p> <p>\u4f7f\u7528\u4ee5\u4e0b\u547d\u4ee4\u6253\u5f00 Windows \u8282\u70b9\u7684 <code>TESTSIGNING</code> \u542f\u52a8\u9879\uff0c\u6267\u884c\u6210\u529f\u540e\u9700\u8981\u91cd\u542f\u7cfb\u7edf\u751f\u6548\uff1a</p> <pre><code>bcdedit /set LOADOPTIONS DISABLE_INTEGRITY_CHECKS\nbcdedit /set TESTSIGNING ON\nbcdedit /set nointegritychecks ON\n</code></pre> <p>\u5728 Windows \u8282\u70b9\u4e0b\u8f7d Windows \u5b89\u88c5\u5305\u5e76\u89e3\u538b\u5b89\u88c5\u3002</p> <p>\u5b89\u88c5\u5b8c\u6210\u540e\u786e\u8ba4\u670d\u52a1\u6b63\u5e38\u8fd0\u884c\uff1a</p> <pre><code>PS &gt; Get-Service | findstr ovs\nRunning  ovsdb-server  Open vSwitch DB Service\nRunning  ovs-vswitchd  Open vSwitch Service\n</code></pre>"},{"location":"advance/windows/#kube-ovn","title":"\u5b89\u88c5 Kube-OVN","text":"<p>\u5728 Windows \u8282\u70b9\u4e0b\u8f7d\u5b89\u88c5\u811a\u672c install.ps1\u3002</p> <p>\u8865\u5145\u76f8\u5173\u53c2\u6570\u5e76\u6267\u884c\uff1a</p> <pre><code>.\\install.ps1 -KubeConfig C:\\k\\admin.conf -ApiServer https://192.168.140.180:6443 -ServiceCIDR 10.96.0.0/12\n</code></pre> <p>\u9ed8\u8ba4\u60c5\u51b5\u4e0b, Kube-OVN \u4f7f\u7528\u8282\u70b9 IP \u6240\u5728\u7684\u7f51\u5361\u4f5c\u4e3a\u96a7\u9053\u63a5\u53e3\u3002 \u5982\u679c\u9700\u8981\u4f7f\u7528\u5176\u5b83\u7f51\u5361\uff0c\u9700\u8981\u5728\u5b89\u88c5\u524d\u7ed9\u8282\u70b9\u6dfb\u52a0\u6307\u5b9a\u7684 Annotation\uff0c\u5982 <code>ovn.kubernetes.io/tunnel_interface=Ethernet1</code>\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/with-bgp/","title":"BGP \u652f\u6301","text":"<p>Kube-OVN \u652f\u6301\u5c06 Pod \u6216 Subnet \u7684 IP \u5730\u5740\u901a\u8fc7 BGP \u534f\u8bae\u5411\u5916\u90e8\u8fdb\u884c\u8def\u7531\u5e7f\u64ad\uff0c\u4ece\u800c\u4f7f\u5f97 Pod IP \u53ef\u4ee5\u76f4\u63a5\u5bf9\u5916\u66b4\u9732\u3002 \u5982\u679c\u9700\u8981\u4f7f\u7528\u8be5\u529f\u80fd\uff0c\u9700\u8981\u5728\u7279\u5b9a\u8282\u70b9\u5b89\u88c5 <code>kube-ovn-speaker</code> \u5e76\u5bf9\u9700\u8981\u5bf9\u5916\u66b4\u9732\u7684 Pod \u6216 Subnet \u589e\u52a0\u5bf9\u5e94\u7684 annotation\u3002</p>"},{"location":"advance/with-bgp/#kube-ovn-speaker","title":"\u5b89\u88c5 kube-ovn-speaker","text":"<p><code>kube-ovn-speaker</code> \u5185\u4f7f\u7528 GoBGP \u5bf9\u5916\u53d1\u5e03\u8def\u7531\u4fe1\u606f\uff0c\u5e76\u5c06\u8bbf\u95ee\u66b4\u9732\u5730\u5740\u7684\u4e0b\u4e00\u8df3\u8def\u7531\u6307\u5411\u81ea\u8eab\u3002</p> <p>\u7531\u4e8e\u90e8\u7f72 <code>kube-ovn-speaker</code> \u7684\u8282\u70b9\u9700\u8981\u627f\u62c5\u56de\u7a0b\u6d41\u91cf\uff0c\u56e0\u6b64\u9700\u8981\u9009\u62e9\u7279\u5b9a\u8282\u70b9\u8fdb\u884c\u90e8\u7f72\uff1a</p> <pre><code>kubectl label nodes speaker-node-1 ovn.kubernetes.io/bgp=true\nkubectl label nodes speaker-node-2 ovn.kubernetes.io/bgp=true\n</code></pre> <p>\u5f53\u5b58\u5728\u591a\u4e2a kube-ovn-speaker \u5b9e\u4f8b\u65f6\uff0c\u6bcf\u4e2a\u5b9e\u4f8b\u90fd\u4f1a\u5bf9\u5916\u53d1\u5e03\u8def\u7531\uff0c\u4e0a\u6e38\u8def\u7531\u5668\u9700\u8981\u652f\u6301\u591a\u8def\u5f84 ECMP\u3002</p> <p>\u4e0b\u8f7d\u5bf9\u5e94 yaml:</p> <pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/yamls/speaker.yaml\n</code></pre> <p>\u4fee\u6539 yaml \u5185\u76f8\u5e94\u914d\u7f6e\uff1a</p> <pre><code>--neighbor-address=10.32.32.1\n--neighbor-as=65030\n--cluster-as=65000\n</code></pre> <ul> <li><code>neighbor-address</code>: BGP Peer \u7684\u5730\u5740\uff0c\u901a\u5e38\u4e3a\u8def\u7531\u5668\u7f51\u5173\u5730\u5740\u3002</li> <li><code>neighbor-as</code>: BGP Peer \u7684 AS \u53f7\u3002</li> <li><code>cluster-as</code>: \u5bb9\u5668\u7f51\u7edc\u7684 AS \u53f7\u3002</li> </ul> <p>\u90e8\u7f72 yaml:</p> <pre><code>kubectl apply -f speaker.yaml\n</code></pre>"},{"location":"advance/with-bgp/#podsubnet","title":"\u53d1\u5e03 Pod/Subnet \u8def\u7531","text":"<p>\u5982\u9700\u4f7f\u7528 BGP \u5bf9\u5916\u53d1\u5e03\u8def\u7531\uff0c\u9996\u5148\u9700\u8981\u5c06\u5bf9\u5e94 Subnet \u7684 <code>natOutgoing</code> \u8bbe\u7f6e\u4e3a <code>false</code>\uff0c\u4f7f\u5f97 Pod IP \u53ef\u4ee5\u76f4\u63a5\u8fdb\u5165\u5e95\u5c42\u7f51\u7edc\u3002</p> <p>\u589e\u52a0 annotation \u5bf9\u5916\u53d1\u5e03\uff1a</p> <pre><code>kubectl annotate pod sample ovn.kubernetes.io/bgp=true\nkubectl annotate subnet ovn-default ovn.kubernetes.io/bgp=true\n</code></pre> <p>\u5220\u9664 annotation \u53d6\u6d88\u53d1\u5e03\uff1a</p> <pre><code>kubectl annotate pod perf-ovn-xzvd4 ovn.kubernetes.io/bgp-\nkubectl annotate subnet ovn-default ovn.kubernetes.io/bgp-\n</code></pre>"},{"location":"advance/with-bgp/#bgp_1","title":"BGP \u9ad8\u7ea7\u9009\u9879","text":"<p><code>kube-ovn-speaker</code> \u652f\u6301\u66f4\u591a BGP \u53c2\u6570\u8fdb\u884c\u9ad8\u7ea7\u914d\u7f6e\uff0c\u7528\u6237\u53ef\u6839\u636e\u81ea\u5df1\u7f51\u7edc\u73af\u5883\u8fdb\u884c\u8c03\u6574\uff1a</p> <ul> <li><code>announce-cluster-ip</code>: \u662f\u5426\u5bf9\u5916\u53d1\u5e03 Service \u8def\u7531\uff0c\u9ed8\u8ba4\u4e3a <code>false</code>\u3002</li> <li><code>auth-password</code>: BGP peer \u7684\u8bbf\u95ee\u5bc6\u7801\u3002</li> <li><code>holdtime</code>: BGP \u90bb\u5c45\u95f4\u7684\u5fc3\u8df3\u63a2\u6d4b\u65f6\u95f4\uff0c\u8d85\u8fc7\u6539\u65f6\u95f4\u6ca1\u6709\u6d88\u606f\u7684\u90bb\u5c45\u5c06\u4f1a\u88ab\u79fb\u9664\uff0c\u9ed8\u8ba4\u4e3a 90 \u79d2\u3002</li> <li><code>graceful-restart</code>: \u662f\u5426\u542f\u7528 BGP Graceful Restart\u3002</li> <li><code>graceful-restart-time</code>: BGP Graceful restart time \u53ef\u53c2\u8003 RFC4724 3\u3002</li> <li><code>graceful-restart-deferral-time</code>: BGP Graceful restart deferral time \u53ef\u53c2\u8003 RFC4724 4.1\u3002</li> <li><code>passivemode</code>: Speaker \u8fd0\u884c\u5728 passive \u6a21\u5f0f\uff0c\u4e0d\u4e3b\u52a8\u8fde\u63a5 peer\u3002</li> <li><code>ebgp-multihop</code>: ebgp ttl \u9ed8\u8ba4\u503c\u4e3a 1\u3002</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/with-cilium/","title":"Cilium \u96c6\u6210","text":"<p>Cilium \u662f\u4e00\u6b3e\u57fa\u4e8e eBPF \u7684\u7f51\u7edc\u548c\u5b89\u5168\u7ec4\u4ef6\uff0cKube-OVN \u5229\u7528\u5176\u4e2d\u7684 CNI Chaining \u6a21\u5f0f\u6765\u5bf9\u5df2\u6709\u529f\u80fd\u8fdb\u884c\u589e\u5f3a\u3002 \u7528\u6237\u53ef\u4ee5\u540c\u65f6\u4f7f\u7528 Kube-OVN \u4e30\u5bcc\u7684\u7f51\u7edc\u62bd\u8c61\u80fd\u529b\u548c eBPF \u5e26\u6765\u7684\u76d1\u63a7\u548c\u5b89\u5168\u80fd\u529b\u3002</p> <p>\u901a\u8fc7\u96c6\u6210 Cilium\uff0cKube-OVN \u7528\u6237\u53ef\u4ee5\u83b7\u5f97\u5982\u4e0b\u589e\u76ca\uff1a</p> <ul> <li>\u66f4\u4e30\u5bcc\u9ad8\u6548\u7684\u5b89\u5168\u7b56\u7565\u3002</li> <li>\u57fa\u4e8e Hubble \u7684\u76d1\u63a7\u89c6\u56fe\u3002</li> </ul> <p></p>"},{"location":"advance/with-cilium/#_1","title":"\u524d\u63d0\u6761\u4ef6","text":"<ol> <li>Linux \u5185\u6838\u7248\u672c\u9ad8\u4e8e 4.19 \u6216\u5176\u4ed6\u517c\u5bb9\u5185\u6838\u4ee5\u83b7\u5f97\u5b8c\u6574 eBPF \u80fd\u529b\u652f\u6301\u3002</li> <li>\u63d0\u524d\u90e8\u7f72 Helm \u4e3a\u5b89\u88c5 Cilium \u505a\u51c6\u5907\uff0c\u90e8\u7f72 Helm \u8bf7\u53c2\u8003 Installing Helm\u3002</li> </ol>"},{"location":"advance/with-cilium/#kube-ovn","title":"\u914d\u7f6e Kube-OVN","text":"<p>\u4e3a\u4e86\u5145\u5206\u4f7f\u7528 Cilium \u7684\u5b89\u5168\u80fd\u529b\uff0c\u9700\u8981\u5173\u95ed Kube-OVN \u5185\u7684 <code>networkpolicy</code> \u529f\u80fd\uff0c\u5e76\u8c03\u6574 CNI \u914d\u7f6e\u4f18\u5148\u7ea7\u3002</p> <p>\u5728 <code>install.sh</code> \u811a\u672c\u91cc\u4fee\u6539\u4e0b\u5217\u53d8\u91cf\uff1a</p> <pre><code>ENABLE_NP=false\nCNI_CONFIG_PRIORITY=10\n</code></pre> <p>\u82e5\u5df2\u90e8\u7f72\u5b8c\u6210\uff0c\u53ef\u901a\u8fc7\u4fee\u6539 <code>kube-ovn-controller</code> \u7684\u542f\u52a8\u53c2\u6570\u8fdb\u884c\u8c03\u6574 <code>networkpolicy</code>\uff1a</p> <pre><code>args:\n- --enable-np=false\n</code></pre> <p>\u4fee\u6539 <code>kube-ovn-cni</code> \u542f\u52a8\u53c2\u6570\u8c03\u6574 CNI \u914d\u7f6e\u4f18\u5148\u7ea7\uff1a</p> <pre><code>args:\n- --cni-conf-name=10-kube-ovn.conflist\n</code></pre> <p>\u5728\u6bcf\u4e2a\u8282\u70b9\u8c03\u6574 Kube-OVN \u914d\u7f6e\u6587\u4ef6\u540d\u79f0\uff0c\u4ee5\u4fbf\u4f18\u5148\u4f7f\u7528 Cilium \u8fdb\u884c\u64cd\u4f5c\uff1a</p> <pre><code>mv /etc/cni/net.d/01-kube-ovn.conflist /etc/cni/net.d/10-kube-ovn.conflist\n</code></pre>"},{"location":"advance/with-cilium/#cilium_1","title":"\u90e8\u7f72 Cilium","text":"<p>\u521b\u5efa <code>chaining.yaml</code> \u914d\u7f6e\u6587\u4ef6\uff0c\u4f7f\u7528 Cilium \u7684 generic-veth \u6a21\u5f0f\uff1a</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: cni-configuration\nnamespace: kube-system\ndata:\ncni-config: |-\n{\n\"name\": \"generic-veth\",\n\"cniVersion\": \"0.3.1\",\n\"plugins\": [\n{\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\",\n\"ipam\": {\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\"\n}\n},\n{\n\"type\": \"portmap\",\n\"snat\": true,\n\"capabilities\": {\"portMappings\": true}\n},\n{\n\"type\": \"cilium-cni\"\n}\n]\n}\n</code></pre> <p>\u5b89\u88c5\u914d\u7f6e\u6587\u4ef6\uff1a</p> <pre><code>kubectl apply -f chaining.yaml\n</code></pre> <p>\u4f7f\u7528 Helm \u90e8\u7f72 Cilium\uff1a</p> <pre><code>helm repo add cilium https://helm.cilium.io/\nhelm install cilium cilium/cilium --version 1.11.6 \\\n--namespace kube-system \\\n--set cni.chainingMode=generic-veth \\\n--set cni.customConf=true \\\n--set cni.configMap=cni-configuration \\\n--set tunnel=disabled \\\n--set enableIPv4Masquerade=false \\\n--set enableIdentityMark=false </code></pre> <p>\u786e\u8ba4 Cilium \u5b89\u88c5\u6210\u529f\uff1a</p> <pre><code># cilium  status\n/\u00af\u00af\\\n/\u00af\u00af\\__/\u00af\u00af\\    Cilium:         OK\n \\__/\u00af\u00af\\__/    Operator:       OK\n /\u00af\u00af\\__/\u00af\u00af\\    Hubble:         disabled\n \\__/\u00af\u00af\\__/    ClusterMesh:    disabled\n    \\__/\n\nDaemonSet         cilium             Desired: 2, Ready: 2/2, Available: 2/2\nDeployment        cilium-operator    Desired: 2, Ready: 2/2, Available: 2/2\nContainers:       cilium             Running: 2\ncilium-operator    Running: 2\nCluster Pods:     8/11 managed by Cilium\nImage versions    cilium             quay.io/cilium/cilium:v1.10.5@sha256:0612218e28288db360c63677c09fafa2d17edda4f13867bcabf87056046b33bb: 2\ncilium-operator    quay.io/cilium/operator-generic:v1.10.5@sha256:2d2f730f219d489ff0702923bf24c0002cd93eb4b47ba344375566202f56d972: 2\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/with-openstack/","title":"OpenStack \u96c6\u6210","text":"<p>\u5728\u4e00\u4e9b\u60c5\u51b5\u4e0b\uff0c\u7528\u6237\u9700\u8981\u4f7f\u7528 OpenStack \u8fd0\u884c\u865a\u62df\u673a\uff0c\u4f7f\u7528 Kubernetes \u8fd0\u884c\u5bb9\u5668\uff0c\u5e76\u9700\u8981\u5bb9\u5668\u548c\u865a\u673a\u4e4b\u95f4\u7f51\u7edc\u4e92\u901a\u5e76\u5904\u4e8e\u7edf\u4e00\u63a7\u5236\u5e73\u9762\u4e0b\u3002\u5982\u679c OpenStack Neutron \u4fa7\u540c\u6837\u4f7f\u7528 OVN \u4f5c\u4e3a\u5e95\u5c42\u7f51\u7edc\u63a7\u5236\uff0c\u90a3\u4e48 Kube-OVN \u53ef\u4ee5\u4f7f\u7528\u96c6\u7fa4\u4e92\u8054\u548c\u5171\u4eab\u5e95\u5c42 OVN \u4e24\u79cd\u65b9\u5f0f\u6253\u901a OpenStack \u548c Kubernetes \u7684\u7f51\u7edc\u3002</p>"},{"location":"advance/with-openstack/#_1","title":"\u96c6\u7fa4\u4e92\u8054","text":"<p>\u8be5\u6a21\u5f0f\u548c\u4f7f\u7528 OVN-IC \u8fdb\u884c\u591a\u96c6\u7fa4\u4e92\u8054\u6253\u901a\u4e24\u4e2a Kubernetes \u96c6\u7fa4\u7f51\u7edc\u65b9\u5f0f\u7c7b\u4f3c\uff0c\u53ea\u4e0d\u8fc7\u5c06\u96c6\u7fa4\u4e24\u7aef\u6362\u6210 OpenStack \u548c Kubernetes\u3002</p>"},{"location":"advance/with-openstack/#_2","title":"\u524d\u63d0\u6761\u4ef6","text":"<ol> <li>\u81ea\u52a8\u4e92\u8054\u6a21\u5f0f\u4e0b OpenStack \u548c Kubernetes \u5185\u7684\u5b50\u7f51 CIDR \u4e0d\u80fd\u76f8\u4e92\u91cd\u53e0\u3002</li> <li>\u9700\u8981\u5b58\u5728\u4e00\u7ec4\u673a\u5668\u53ef\u4ee5\u88ab\u6bcf\u4e2a\u96c6\u7fa4\u901a\u8fc7\u7f51\u7edc\u8bbf\u95ee\uff0c\u7528\u6765\u90e8\u7f72\u8de8\u96c6\u7fa4\u4e92\u8054\u7684\u63a7\u5236\u5668\u3002</li> <li>\u6bcf\u4e2a\u96c6\u7fa4\u9700\u8981\u6709\u4e00\u7ec4\u53ef\u4ee5\u901a\u8fc7 IP \u8fdb\u884c\u8de8\u96c6\u7fa4\u4e92\u8bbf\u7684\u673a\u5668\u4f5c\u4e3a\u4e4b\u540e\u7684\u7f51\u5173\u8282\u70b9\u3002</li> <li>\u8be5\u65b9\u6848\u53ea\u6253\u901a Kubernetes \u9ed8\u8ba4\u5b50\u7f51\u548c OpenStack \u7684\u9009\u5b9a VPC\u3002</li> </ol>"},{"location":"advance/with-openstack/#ovn-ic","title":"\u90e8\u7f72 OVN-IC \u6570\u636e\u5e93","text":"<p>\u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u542f\u52a8 <code>OVN-IC</code> \u6570\u636e\u5e93\uff1a</p> <pre><code>docker run --name=ovn-ic-db -d --network=host -v /etc/ovn/:/etc/ovn -v /var/run/ovn:/var/run/ovn -v /var/log/ovn:/var/log/ovn kubeovn/kube-ovn:v1.13.0 bash start-ic-db.sh\n</code></pre>"},{"location":"advance/with-openstack/#kubernetes","title":"Kubernetes \u4fa7\u64cd\u4f5c","text":"<p>\u5728 <code>kube-system</code> Namespace \u4e0b\u521b\u5efa <code>ovn-ic-config</code> ConfigMap\uff1a</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: ovn-ic-config\nnamespace: kube-system\ndata:\nenable-ic: \"true\"\naz-name: \"az1\" ic-db-host: \"192.168.65.3\"\nic-nb-port: \"6645\" ic-sb-port: \"6646\"\ngw-nodes: \"az1-gw\"\nauto-route: \"true\"\n</code></pre> <ul> <li><code>enable-ic</code>: \u662f\u5426\u5f00\u542f\u96c6\u7fa4\u4e92\u8054\u3002</li> <li><code>az-name</code>: \u533a\u5206\u4e0d\u540c\u96c6\u7fa4\u7684\u96c6\u7fa4\u540d\u79f0\uff0c\u6bcf\u4e2a\u4e92\u8054\u96c6\u7fa4\u9700\u4e0d\u540c\u3002</li> <li><code>ic-db-host</code>: \u90e8\u7f72 <code>OVN-IC</code> \u6570\u636e\u5e93\u7684\u8282\u70b9\u5730\u5740\u3002</li> <li><code>ic-nb-port</code>: <code>OVN-IC</code> \u5317\u5411\u6570\u636e\u5e93\u7aef\u53e3\uff0c\u9ed8\u8ba4\u4e3a 6645\u3002</li> <li><code>ic-sb-port</code>: <code>OVN-IC</code> \u5357\u5411\u6570\u636e\u5e93\u7aef\u53e3\uff0c\u9ed8\u8ba4\u4e3a 6646\u3002</li> <li><code>gw-nodes</code>: \u96c6\u7fa4\u4e92\u8054\u4e2d\u627f\u62c5\u7f51\u5173\u5de5\u4f5c\u7684\u8282\u70b9\u540d\uff0c\u9017\u53f7\u5206\u9694\u3002</li> <li><code>auto-route</code>: \u662f\u5426\u81ea\u52a8\u5bf9\u5916\u53d1\u5e03\u548c\u5b66\u4e60\u8def\u7531\u3002</li> </ul>"},{"location":"advance/with-openstack/#openstack_1","title":"OpenStack \u4fa7\u64cd\u4f5c","text":"<p>\u521b\u5efa\u548c Kubernetes \u4e92\u8054\u7684\u903b\u8f91\u8def\u7531\u5668\uff1a</p> <pre><code># openstack router create router0\n# openstack router list\n+--------------------------------------+---------+--------+-------+----------------------------------+\n| ID                                   | Name    | Status | State | Project                          |\n+--------------------------------------+---------+--------+-------+----------------------------------+\n| d5b38655-249a-4192-8046-71aa4d2b4af1 | router0 | ACTIVE | UP    | 98a29ab7388347e7b5ff8bdd181ba4f9 |\n+--------------------------------------+---------+--------+-------+----------------------------------+\n</code></pre> <p>\u5728 OpenStack \u5185\u7684 OVN \u5317\u5411\u6570\u636e\u5e93\u4e2d\u8bbe\u7f6e\u53ef\u7528\u533a\u540d\u5b57\uff0c\u8be5\u540d\u79f0\u9700\u548c\u5176\u4ed6\u4e92\u8054\u96c6\u7fa4\u4e0d\u540c\uff1a</p> <pre><code>ovn-nbctl set NB_Global . name=op-az\n</code></pre> <p>\u5728\u53ef\u8bbf\u95ee <code>OVN-IC</code> \u6570\u636e\u5e93\u7684\u8282\u70b9\u542f\u52a8 <code>OVN-IC</code> \u63a7\u5236\u5668\uff1a</p> <pre><code>/usr/share/ovn/scripts/ovn-ctl --ovn-ic-nb-db=tcp:192.168.65.3:6645 \\\n--ovn-ic-sb-db=tcp:192.168.65.3:6646 \\\n--ovn-northd-nb-db=unix:/run/ovn/ovnnb_db.sock \\\n--ovn-northd-sb-db=unix:/run/ovn/ovnsb_db.sock \\\nstart_ic\n</code></pre> <ul> <li><code>ovn-ic-nb-db</code>\uff0c<code>ovn-ic-sb-db</code>: OVN-IC \u5317\u5411\u6570\u636e\u5e93\u548c\u5357\u5411\u6570\u636e\u5e93\u5730\u5740\u3002</li> <li><code>ovn-northd-nb-db</code>\uff0c <code>ovn-northd-sb-db</code>: \u5f53\u524d\u96c6\u7fa4 OVN \u5317\u5411\u6570\u636e\u5e93\u548c\u5357\u5411\u6570\u636e\u5730\u5740\u3002</li> </ul> <p>\u914d\u7f6e\u4e92\u8054\u7f51\u5173\u8282\u70b9\uff1a</p> <pre><code>ovs-vsctl set open_vswitch . external_ids:ovn-is-interconn=true\n</code></pre> <p>\u63a5\u4e0b\u6765\u9700\u8981\u5728 OpenStack \u7684 OVN \u5185\u8fdb\u884c\u64cd\u4f5c\u521b\u5efa\u903b\u8f91\u62d3\u6251\u3002</p> <p>\u8fde\u63a5 <code>ts</code> \u4e92\u8054\u4ea4\u6362\u673a\u548c <code>router0</code> \u903b\u8f91\u8def\u7531\u5668\uff0c\u5e76\u8bbe\u7f6e\u76f8\u5173\u89c4\u5219\uff1a</p> <pre><code>ovn-nbctl lrp-add router0 lrp-router0-ts 00:02:ef:11:39:4f 169.254.100.73/24\novn-nbctl lsp-add ts lsp-ts-router0 -- lsp-set-addresses lsp-ts-router0 router \\\n-- lsp-set-type lsp-ts-router0 router \\\n-- lsp-set-options lsp-ts-router0  router-port=lrp-router0-ts\novn-nbctl lrp-set-gateway-chassis lrp-router0-ts {gateway chassis} 1000\novn-nbctl set NB_Global . options:ic-route-adv=true options:ic-route-learn=true\n</code></pre> <p>\u9a8c\u8bc1\u5df2\u5b66\u4e60\u5230 Kubernetes \u8def\u7531\u89c4\u5219\uff1a</p> <pre><code># ovn-nbctl lr-route-list router0\nIPv4 Routes\n                10.0.0.22            169.254.100.34 dst-ip (learned)\n10.16.0.0/16            169.254.100.34 dst-ip (learned)\n</code></pre> <p>\u63a5\u4e0b\u6765\u53ef\u4ee5\u5728 <code>router0</code> \u7f51\u7edc\u4e0b\u521b\u5efa\u865a\u673a\u9a8c\u8bc1\u662f\u5426\u53ef\u4ee5\u548c Kubernetes \u4e0b Pod \u4e92\u901a\u3002</p>"},{"location":"advance/with-openstack/#ovn","title":"\u5171\u4eab\u5e95\u5c42 OVN","text":"<p>\u5728\u8be5\u65b9\u6848\u4e0b\uff0cOpenStack \u548c Kubernetes \u5171\u4eab\u4f7f\u7528\u540c\u4e00\u4e2a OVN\uff0c\u56e0\u6b64\u53ef\u4ee5\u5c06\u4e24\u8005\u7684 VPC \u548c Subnet \u7b49\u6982\u5ff5\u62c9\u9f50\uff0c\u5b9e\u73b0\u66f4\u597d\u7684\u63a7\u5236\u548c\u4e92\u8054\u3002</p> <p>\u5728\u8be5\u6a21\u5f0f\u4e0b\u6211\u4eec\u6b63\u5e38\u4f7f\u7528 Kube-OVN \u90e8\u7f72 OVN\uff0cOpenStack \u4fee\u6539 Neutron \u914d\u7f6e\u5b9e\u73b0\u8fde\u63a5\u540c\u4e00\u4e2a OVN \u6570\u636e\u5e93\u3002OpenStack \u9700\u4f7f\u7528 networking-ovn \u4f5c\u4e3a Neutron \u540e\u7aef\u5b9e\u73b0\u3002</p>"},{"location":"advance/with-openstack/#neutron","title":"Neutron \u914d\u7f6e\u4fee\u6539","text":"<p>\u4fee\u6539 Neutron \u914d\u7f6e\u6587\u4ef6 <code>/etc/neutron/plugins/ml2/ml2_conf.ini</code>\uff1a</p> <pre><code>[ovn]\n...\novn_nb_connection = tcp:[192.168.137.176]:6641,tcp:[192.168.137.177]:6641,tcp:[192.168.137.178]:6641\novn_sb_connection = tcp:[192.168.137.176]:6642,tcp:[192.168.137.177]:6642,tcp:[192.168.137.178]:6642\novn_l3_scheduler = OVN_L3_SCHEDULER\n</code></pre> <ul> <li><code>ovn_nb_connection</code>\uff0c <code>ovn_sb_connection</code>: \u5730\u5740\u9700\u4fee\u6539\u4e3a Kube-OVN \u90e8\u7f72 <code>ovn-central</code> \u8282\u70b9\u7684\u5730\u5740\u3002</li> </ul> <p>\u4fee\u6539\u6bcf\u4e2a\u8282\u70b9\u7684 OVS \u914d\u7f6e\uff1a</p> <pre><code>ovs-vsctl set open . external-ids:ovn-remote=tcp:[192.168.137.176]:6642,tcp:[192.168.137.177]:6642,tcp:[192.168.137.178]:6642\novs-vsctl set open . external-ids:ovn-encap-type=geneve\novs-vsctl set open . external-ids:ovn-encap-ip=192.168.137.200\n</code></pre> <ul> <li><code>external-ids:ovn-remote</code>: \u5730\u5740\u9700\u4fee\u6539\u4e3a Kube-OVN \u90e8\u7f72 <code>ovn-central</code> \u8282\u70b9\u7684\u5730\u5740\u3002</li> <li><code>ovn-encap-ip</code>: \u4fee\u6539\u4e3a\u5f53\u524d\u8282\u70b9\u7684 IP \u5730\u5740\u3002</li> </ul>"},{"location":"advance/with-openstack/#kubernetes-openstack","title":"\u5728 Kubernetes \u4e2d\u4f7f\u7528 OpenStack \u5185\u8d44\u6e90","text":"<p>\u63a5\u4e0b\u6765\u4ecb\u7ecd\u5982\u4f55\u5728 Kubernetes \u4e2d\u67e5\u8be2 OpenStack \u7684\u7f51\u7edc\u8d44\u6e90\u5e76\u5728 OpenStack \u7684\u5b50\u7f51\u4e2d\u521b\u5efa Pod\u3002</p> <p>\u67e5\u8be2 OpenStack \u4e2d\u5df2\u6709\u7684\u7f51\u7edc\u8d44\u6e90\uff0c\u5982\u4e0b\u8d44\u6e90\u5df2\u7ecf\u9884\u5148\u521b\u5efa\u5b8c\u6210\uff1a</p> <pre><code># openstack router list\n+--------------------------------------+---------+--------+-------+----------------------------------+\n| ID                                   | Name    | Status | State | Project                          |\n+--------------------------------------+---------+--------+-------+----------------------------------+\n| 22040ed5-0598-4f77-bffd-e7fd4db47e93 | router0 | ACTIVE | UP    | 62381a21d569404aa236a5dd8712449c |\n+--------------------------------------+---------+--------+-------+----------------------------------+\n# openstack network list\n+--------------------------------------+----------+--------------------------------------+\n| ID                                   | Name     | Subnets                              |\n+--------------------------------------+----------+--------------------------------------+\n| cd59e36a-37db-4c27-b709-d35379a7920f | provider | 01d73d9f-fdaa-426c-9b60-aa34abbfacae |\n+--------------------------------------+----------+--------------------------------------+\n# openstack subnet list\n+--------------------------------------+-------------+--------------------------------------+----------------+\n| ID                                   | Name        | Network                              | Subnet         |\n+--------------------------------------+-------------+--------------------------------------+----------------+\n| 01d73d9f-fdaa-426c-9b60-aa34abbfacae | provider-v4 | cd59e36a-37db-4c27-b709-d35379a7920f | 192.168.1.0/24 |\n+--------------------------------------+-------------+--------------------------------------+----------------+\n# openstack server list\n+--------------------------------------+-------------------+--------+-----------------------+--------+--------+\n| ID                                   | Name              | Status | Networks              | Image  | Flavor |\n+--------------------------------------+-------------------+--------+-----------------------+--------+--------+\n| 8433d622-a8d6-41a7-8b31-49abfd64f639 | provider-instance | ACTIVE | provider=192.168.1.61 | ubuntu | m1     |\n+--------------------------------------+-------------------+--------+-----------------------+--------+--------+\n</code></pre> <p>\u5728 Kubernetes \u4fa7\uff0c\u67e5\u8be2 VPC \u8d44\u6e90\uff1a</p> <pre><code># kubectl get vpc\nNAME                                           STANDBY   SUBNETS\nneutron-22040ed5-0598-4f77-bffd-e7fd4db47e93   true      [\"neutron-cd59e36a-37db-4c27-b709-d35379a7920f\"]\novn-cluster                                    true      [\"join\",\"ovn-default\"]\n</code></pre> <p><code>neutron-22040ed5-0598-4f77-bffd-e7fd4db47e93</code> \u4e3a\u4ece OpenStack \u540c\u6b65\u8fc7\u6765\u7684 VPC \u8d44\u6e90\u3002</p> <p>\u63a5\u4e0b\u6765\u53ef\u4ee5\u6309\u7167 Kube-OVN \u539f\u751f\u7684 VPC \u548c Subnet \u64cd\u4f5c\u521b\u5efa Pod \u5e76\u8fd0\u884c\u3002</p> <p>VPC, Subnet \u7ed1\u5b9a Namespace <code>net2</code>\uff0c\u5e76\u521b\u5efa Pod:</p> <pre><code>apiVersion: v1\nkind: Namespace\nmetadata:\nname: net2\n---\napiVersion: kubeovn.io/v1\nkind: Vpc\nmetadata:\ncreationTimestamp: \"2021-06-20T13:34:11Z\"\ngeneration: 2\nlabels:\novn.kubernetes.io/vpc_external: \"true\"\nname: neutron-22040ed5-0598-4f77-bffd-e7fd4db47e93\nresourceVersion: \"583728\"\nuid: 18d4c654-f511-4def-a3a0-a6434d237c1e\nspec:\nnamespaces:\n- net2\n---\nkind: Subnet\napiVersion: kubeovn.io/v1\nmetadata:\nname: net2\nspec:\nvpc: neutron-22040ed5-0598-4f77-bffd-e7fd4db47e93\nnamespaces:\n- net2\ncidrBlock: 12.0.1.0/24\nnatOutgoing: false\n---\napiVersion: v1\nkind: Pod\nmetadata:\nname: ubuntu\nnamespace: net2\nspec:\ncontainers:\n- image: docker.io/kubeovn/kube-ovn:v1.8.0\ncommand:\n- \"sleep\"\n- \"604800\"\nimagePullPolicy: IfNotPresent\nname: ubuntu\nrestartPolicy: Always\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/with-ovn-ic/","title":"\u4f7f\u7528 OVN-IC \u8fdb\u884c\u591a\u96c6\u7fa4\u4e92\u8054","text":"<p>Kube-OVN \u652f\u6301\u901a\u8fc7 OVN-IC \u5c06\u4e24\u4e2a Kubernetes \u96c6\u7fa4 Pod \u7f51\u7edc\u6253\u901a\uff0c\u6253\u901a\u540e\u7684\u4e24\u4e2a\u96c6\u7fa4\u5185\u7684 Pod \u53ef\u4ee5\u901a\u8fc7 Pod IP \u8fdb\u884c\u76f4\u63a5\u901a\u4fe1\u3002 Kube-OVN \u4f7f\u7528\u96a7\u9053\u5bf9\u8de8\u96c6\u7fa4\u6d41\u91cf\u8fdb\u884c\u5c01\u88c5\uff0c\u4e24\u4e2a\u96c6\u7fa4\u4e4b\u95f4\u53ea\u8981\u5b58\u5728\u4e00\u7ec4 IP \u53ef\u8fbe\u7684\u673a\u5668\u5373\u53ef\u5b8c\u6210\u5bb9\u5668\u7f51\u7edc\u7684\u4e92\u901a\u3002</p> <p>\u8be5\u6a21\u5f0f\u7684\u591a\u96c6\u7fa4\u4e92\u8054\u4e3a Overlay \u7f51\u7edc\u529f\u80fd\uff0cUnderlay \u7f51\u7edc\u5982\u679c\u60f3\u8981\u5b9e\u73b0\u96c6\u7fa4\u4e92\u8054\u9700\u8981\u5e95\u5c42\u57fa\u7840\u8bbe\u65bd\u505a\u7f51\u7edc\u6253\u901a\u3002</p> <p></p>"},{"location":"advance/with-ovn-ic/#_1","title":"\u524d\u63d0\u6761\u4ef6","text":"<ol> <li>\u81ea\u52a8\u4e92\u8054\u6a21\u5f0f\u4e0b\u4e0d\u540c\u96c6\u7fa4\u7684\u5b50\u7f51 CIDR \u4e0d\u80fd\u76f8\u4e92\u91cd\u53e0\uff0c\u9ed8\u8ba4\u5b50\u7f51\u9700\u5728\u5b89\u88c5\u65f6\u914d\u7f6e\u4e3a\u4e0d\u91cd\u53e0\u7684\u7f51\u6bb5\u3002\u82e5\u5b58\u5728\u91cd\u53e0\u9700\u53c2\u8003\u540e\u7eed\u624b\u52a8\u4e92\u8054\u8fc7\u7a0b\uff0c\u53ea\u80fd\u5c06\u4e0d\u91cd\u53e0\u7f51\u6bb5\u6253\u901a\u3002</li> <li>\u9700\u8981\u5b58\u5728\u4e00\u7ec4\u673a\u5668\u53ef\u4ee5\u88ab\u6bcf\u4e2a\u96c6\u7fa4\u7684 <code>kube-ovn-controller</code> \u901a\u8fc7 IP \u8bbf\u95ee\uff0c\u7528\u6765\u90e8\u7f72\u8de8\u96c6\u7fa4\u4e92\u8054\u7684\u63a7\u5236\u5668\u3002</li> <li>\u6bcf\u4e2a\u96c6\u7fa4\u9700\u8981\u6709\u4e00\u7ec4\u53ef\u4ee5\u901a\u8fc7 IP \u8fdb\u884c\u8de8\u96c6\u7fa4\u4e92\u8bbf\u7684\u673a\u5668\u4f5c\u4e3a\u4e4b\u540e\u7684\u7f51\u5173\u8282\u70b9\u3002</li> <li>\u8be5\u529f\u80fd\u53ea\u5bf9\u9ed8\u8ba4 VPC \u751f\u6548\uff0c\u7528\u6237\u81ea\u5b9a\u4e49 VPC \u65e0\u6cd5\u4f7f\u7528\u4e92\u8054\u529f\u80fd\u3002</li> </ol>"},{"location":"advance/with-ovn-ic/#ovn-ic_1","title":"\u90e8\u7f72\u5355\u8282\u70b9 OVN-IC \u6570\u636e\u5e93","text":"<p>\u5728\u6bcf\u4e2a\u96c6\u7fa4 <code>kube-ovn-controller</code> \u53ef\u901a\u8fc7 IP \u8bbf\u95ee\u7684\u673a\u5668\u4e0a\u90e8\u7f72 <code>OVN-IC</code> \u6570\u636e\u5e93\uff0c\u8be5\u8282\u70b9\u5c06\u4fdd\u5b58\u5404\u4e2a\u96c6\u7fa4\u540c\u6b65\u4e0a\u6765\u7684\u7f51\u7edc\u914d\u7f6e\u4fe1\u606f\u3002</p> <p>\u90e8\u7f72 <code>docker</code> \u7684\u73af\u5883\u53ef\u4ee5\u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u542f\u52a8 <code>OVN-IC</code> \u6570\u636e\u5e93\uff1a</p> <pre><code>docker run --name=ovn-ic-db -d --network=host --privileged  -v /etc/ovn/:/etc/ovn -v /var/run/ovn:/var/run/ovn -v /var/log/ovn:/var/log/ovn kubeovn/kube-ovn:v1.13.0 bash start-ic-db.sh\n</code></pre> <p>\u5bf9\u4e8e\u90e8\u7f72 <code>containerd</code> \u53d6\u4ee3 <code>docker</code> \u7684\u73af\u5883\u53ef\u4ee5\u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\uff1a</p> <pre><code>ctr -n k8s.io run -d --net-host --privileged --mount=\"type=bind,src=/etc/ovn/,dst=/etc/ovn,options=rbind:rw\" --mount=\"type=bind,src=/var/run/ovn,dst=/var/run/ovn,options=rbind:rw\" --mount=\"type=bind,src=/var/log/ovn,dst=/var/log/ovn,options=rbind:rw\" docker.io/kubeovn/kube-ovn:v1.13.0 ovn-ic-db bash start-ic-db.sh\n</code></pre>"},{"location":"advance/with-ovn-ic/#_2","title":"\u81ea\u52a8\u8def\u7531\u8bbe\u7f6e","text":"<p>\u5728\u81ea\u52a8\u8def\u7531\u8bbe\u7f6e\u4e0b\uff0c\u6bcf\u4e2a\u96c6\u7fa4\u4f1a\u5c06\u81ea\u5df1\u9ed8\u8ba4 VPC \u4e0b Subnet \u7684 CIDR \u4fe1\u606f\u540c\u6b65\u7ed9 <code>OVN-IC</code>\uff0c\u56e0\u6b64\u8981\u786e\u4fdd\u4e24\u4e2a\u96c6\u7fa4\u7684 Subnet CIDR \u4e0d\u5b58\u5728\u91cd\u53e0\u3002</p> <p>\u5728 <code>kube-system</code> Namespace \u4e0b\u521b\u5efa <code>ovn-ic-config</code> ConfigMap\uff1a</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: ovn-ic-config\nnamespace: kube-system\ndata:\nenable-ic: \"true\"\naz-name: \"az1\" ic-db-host: \"192.168.65.3\"\nic-nb-port: \"6645\" ic-sb-port: \"6646\"\ngw-nodes: \"az1-gw\"\nauto-route: \"true\"\n</code></pre> <ul> <li><code>enable-ic</code>: \u662f\u5426\u5f00\u542f\u96c6\u7fa4\u4e92\u8054\u3002</li> <li><code>az-name</code>: \u533a\u5206\u4e0d\u540c\u96c6\u7fa4\u7684\u96c6\u7fa4\u540d\u79f0\uff0c\u6bcf\u4e2a\u4e92\u8054\u96c6\u7fa4\u9700\u4e0d\u540c\u3002</li> <li><code>ic-db-host</code>: \u90e8\u7f72 <code>OVN-IC</code> \u6570\u636e\u5e93\u7684\u8282\u70b9\u5730\u5740\u3002</li> <li><code>ic-nb-port</code>: <code>OVN-IC</code> \u5317\u5411\u6570\u636e\u5e93\u7aef\u53e3\uff0c\u9ed8\u8ba4\u4e3a 6645\u3002</li> <li><code>ic-sb-port</code>: <code>OVN-IC</code> \u5357\u5411\u6570\u636e\u5e93\u7aef\u53e3\uff0c\u9ed8\u8ba4\u4e3a 6646\u3002</li> <li><code>gw-nodes</code>: \u96c6\u7fa4\u4e92\u8054\u4e2d\u627f\u62c5\u7f51\u5173\u5de5\u4f5c\u7684\u8282\u70b9\u540d\uff0c\u9017\u53f7\u5206\u9694\u3002</li> <li><code>auto-route</code>: \u662f\u5426\u81ea\u52a8\u5bf9\u5916\u53d1\u5e03\u548c\u5b66\u4e60\u8def\u7531\u3002</li> </ul> <p>\u6ce8\u610f\uff1a \u4e3a\u4e86\u4fdd\u8bc1\u64cd\u4f5c\u7684\u6b63\u786e\u6027\uff0c<code>ovn-ic-config</code> \u8fd9\u4e2a ConfigMap \u4e0d\u5141\u8bb8\u4fee\u6539\u3002\u5982\u6709\u53c2\u6570\u9700\u8981\u53d8\u66f4\uff0c\u8bf7\u5220\u9664\u8be5 ConfigMap\uff0c\u4fee\u6539\u540e\u518d\u5e94\u7528\u6b64 ConfigMap\u3002</p> <p>\u5728 <code>ovn-ic</code> \u5bb9\u5668\u5185\u901a\u8fc7\u4e0b\u9762\u7684\u547d\u4ee4\u67e5\u770b\u662f\u5426\u5df2\u5efa\u7acb\u4e92\u8054\u903b\u8f91\u4ea4\u6362\u673a <code>ts</code>\uff1a</p> <pre><code># ovn-ic-sbctl show\navailability-zone az1\n    gateway deee03e0-af16-4f45-91e9-b50c3960f809\n        hostname: az1-gw\n        type: geneve\n            ip: 192.168.42.145\n        port ts-az1\n            transit switch: ts\n            address: [\"00:00:00:50:AC:8C 169.254.100.45/24\"]\navailability-zone az2\n    gateway e94cc831-8143-40e3-a478-90352773327b\n        hostname: az2-gw\n        type: geneve\n            ip: 192.168.42.149\n        port ts-az2\n            transit switch: ts\n            address: [\"00:00:00:07:4A:59 169.254.100.63/24\"]\n</code></pre> <p>\u5728\u6bcf\u4e2a\u96c6\u7fa4\u89c2\u5bdf\u903b\u8f91\u8def\u7531\u662f\u5426\u6709\u5b66\u4e60\u5230\u7684\u5bf9\u7aef\u8def\u7531\uff1a</p> <pre><code># kubectl ko nbctl lr-route-list ovn-cluster\nIPv4 Routes\n                10.42.1.1            169.254.100.45 dst-ip (learned)\n10.42.1.3                100.64.0.2 dst-ip\n                10.16.0.2                100.64.0.2 src-ip\n                10.16.0.3                100.64.0.2 src-ip\n                10.16.0.4                100.64.0.2 src-ip\n                10.16.0.6                100.64.0.2 src-ip\n             10.17.0.0/16            169.254.100.45 dst-ip (learned)\n100.65.0.0/16            169.254.100.45 dst-ip (learned)\n</code></pre> <p>\u63a5\u4e0b\u6765\u53ef\u4ee5\u5c1d\u8bd5\u5728\u96c6\u7fa4 1 \u5185\u7684\u4e00\u4e2a Pod \u5185\u76f4\u63a5 <code>ping</code> \u96c6\u7fa4 2 \u5185\u7684\u4e00\u4e2a Pod IP \u89c2\u5bdf\u662f\u5426\u53ef\u4ee5\u8054\u901a\u3002</p> <p>\u5bf9\u4e8e\u67d0\u4e2a\u4e0d\u60f3\u5bf9\u5916\u81ea\u52a8\u53d1\u5e03\u8def\u7531\u7684\u5b50\u7f51\u53ef\u4ee5\u901a\u8fc7\u4fee\u6539 Subnet \u91cc\u7684 <code>disableInterConnection</code> \u6765\u7981\u6b62\u8def\u7531\u5e7f\u64ad\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: no-advertise\nspec:\ncidrBlock: 10.199.0.0/16\ndisableInterConnection: true\n</code></pre>"},{"location":"advance/with-ovn-ic/#_3","title":"\u624b\u52a8\u8def\u7531\u8bbe\u7f6e","text":"<p>\u5bf9\u4e8e\u96c6\u7fa4\u95f4\u5b58\u5728\u91cd\u53e0 CIDR \u53ea\u5e0c\u671b\u505a\u90e8\u5206\u5b50\u7f51\u6253\u901a\u7684\u60c5\u51b5\uff0c\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u6b65\u9aa4\u624b\u52a8\u53d1\u5e03\u5b50\u7f51\u8def\u7531\u3002</p> <p>\u5728 <code>kube-system</code> Namespace \u4e0b\u521b\u5efa <code>ovn-ic-config</code> ConfigMap\uff0c\u5e76\u5c06 <code>auto-route</code> \u8bbe\u7f6e\u4e3a <code>false</code>\uff1a</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: ovn-ic-config\nnamespace: kube-system\ndata:\nenable-ic: \"true\"\naz-name: \"az1\" ic-db-host: \"192.168.65.3\"\nic-nb-port: \"6645\" ic-sb-port: \"6646\"\ngw-nodes: \"az1-gw\"\nauto-route: \"false\"\n</code></pre> <p>\u5728\u6bcf\u4e2a\u96c6\u7fa4\u5206\u522b\u67e5\u770b\u8fdc\u7aef\u903b\u8f91\u7aef\u53e3\u7684\u5730\u5740\uff0c\u7528\u4e8e\u4e4b\u540e\u624b\u52a8\u914d\u7f6e\u8def\u7531\uff1a</p> <pre><code>[root@az1 ~]# kubectl ko nbctl show\nswitch a391d3a1-14a0-4841-9836-4bd930c447fb (ts)\nport ts-az1\n        type: router\n        router-port: az1-ts\n    port ts-az2\n        type: remote\n        addresses: [\"00:00:00:4B:E2:9F 169.254.100.31/24\"]\n\n[root@az2 ~]# kubectl ko nbctl show\nswitch da6138b8-de81-4908-abf9-b2224ec4edf3 (ts)\nport ts-az2\n        type: router\n        router-port: az2-ts\n    port ts-az1\n        type: remote\n        addresses: [\"00:00:00:FB:2A:F7 169.254.100.79/24\"]        </code></pre> <p>\u7531\u4e0a\u8f93\u51fa\u53ef\u77e5\uff0c\u96c6\u7fa4 <code>az1</code> \u5230 \u96c6\u7fa4 <code>az2</code> \u7684\u8fdc\u7aef\u5730\u5740\u4e3a <code>169.254.100.31</code>\uff0c<code>az2</code> \u5230 <code>az1</code> \u7684\u8fdc\u7aef\u5730\u5740\u4e3a <code>169.254.100.79</code>\u3002</p> <p>\u4e0b\u9762\u624b\u52a8\u8bbe\u7f6e\u8def\u7531\uff0c\u5728\u8be5\u4f8b\u5b50\u4e2d\uff0c\u96c6\u7fa4 <code>az1</code> \u5185\u7684\u5b50\u7f51 CIDR \u4e3a <code>10.16.0.0/24</code>\uff0c\u96c6\u7fa4 <code>az2</code> \u5185\u7684\u5b50\u7f51 CIDR \u4e3a <code>10.17.0.0/24</code>\u3002</p> <p>\u5728\u96c6\u7fa4 <code>az1</code> \u8bbe\u7f6e\u5230\u96c6\u7fa4 <code>az2</code> \u7684\u8def\u7531:</p> <pre><code>kubectl ko nbctl lr-route-add ovn-cluster 10.17.0.0/24 169.254.100.31\n</code></pre> <p>\u5728\u96c6\u7fa4 <code>az2</code> \u8bbe\u7f6e\u5230\u96c6\u7fa4 <code>az1</code> \u7684\u8def\u7531:</p> <pre><code>kubectl ko nbctl lr-route-add ovn-cluster 10.16.0.0/24 169.254.100.79\n</code></pre>"},{"location":"advance/with-ovn-ic/#ovn-ic_2","title":"\u9ad8\u53ef\u7528 OVN-IC \u6570\u636e\u5e93\u90e8\u7f72","text":"<p><code>OVN-IC</code> \u6570\u636e\u5e93\u4e4b\u95f4\u53ef\u4ee5\u901a\u8fc7 Raft \u534f\u8bae\u7ec4\u6210\u4e00\u4e2a\u9ad8\u53ef\u7528\u96c6\u7fa4\uff0c\u8be5\u90e8\u7f72\u6a21\u5f0f\u9700\u8981\u81f3\u5c11 3 \u4e2a\u8282\u70b9\u3002</p> <p>\u9996\u5148\u5728\u7b2c\u4e00\u4e2a\u8282\u70b9\u4e0a\u542f\u52a8 <code>OVN-IC</code> \u6570\u636e\u5e93\u7684 leader\u3002</p> <p>\u90e8\u7f72 <code>docker</code> \u73af\u5883\u7684\u7528\u6237\u53ef\u4ee5\u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\uff1a</p> <pre><code>docker run --name=ovn-ic-db -d --network=host --privileged -v /etc/ovn/:/etc/ovn -v /var/run/ovn:/var/run/ovn -v /var/log/ovn:/var/log/ovn -e LOCAL_IP=\"192.168.65.3\"  -e NODE_IPS=\"192.168.65.3,192.168.65.2,192.168.65.1\"   kubeovn/kube-ovn:v1.13.0 bash start-ic-db.sh\n</code></pre> <p>\u5982\u679c\u662f\u90e8\u7f72 <code>containerd</code> \u7684\u7528\u6237\u53ef\u4ee5\u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\uff1a</p> <pre><code>ctr -n k8s.io run -d --net-host --privileged --mount=\"type=bind,src=/etc/ovn/,dst=/etc/ovn,options=rbind:rw\" --mount=\"type=bind,src=/var/run/ovn,dst=/var/run/ovn,options=rbind:rw\" --mount=\"type=bind,src=/var/log/ovn,dst=/var/log/ovn,options=rbind:rw\"  --env=\"NODE_IPS=\"192.168.65.3,192.168.65.2,192.168.65.1\"\" --env=\"LOCAL_IP=\"192.168.65.3\"\" docker.io/kubeovn/kube-ovn:v1.13.0 ovn-ic-db bash start-ic-db.sh\n</code></pre> <ul> <li><code>LOCAL_IP</code>\uff1a \u5f53\u524d\u5bb9\u5668\u6240\u5728\u8282\u70b9 IP \u5730\u5740\u3002</li> <li><code>NODE_IPS</code>\uff1a \u8fd0\u884c <code>OVN-IC</code> \u6570\u636e\u5e93\u7684\u4e09\u4e2a\u8282\u70b9 IP \u5730\u5740\uff0c\u4f7f\u7528\u9017\u53f7\u8fdb\u884c\u5206\u9694\u3002</li> </ul> <p>\u63a5\u4e0b\u6765\uff0c\u5728\u53e6\u5916\u4e24\u4e2a\u8282\u70b9\u90e8\u7f72 <code>OVN-IC</code> \u6570\u636e\u5e93\u7684 follower\u3002</p> <p>\u90e8\u7f72 <code>docker</code> \u73af\u5883\u7684\u7528\u6237\u53ef\u4ee5\u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\uff1a</p> <pre><code>docker run --name=ovn-ic-db -d --network=host --privileged -v /etc/ovn/:/etc/ovn -v /var/run/ovn:/var/run/ovn -v /var/log/ovn:/var/log/ovn -e LOCAL_IP=\"192.168.65.2\"  -e NODE_IPS=\"192.168.65.3,192.168.65.2,192.168.65.1\" -e LEADER_IP=\"192.168.65.3\"  kubeovn/kube-ovn:v1.13.0 bash start-ic-db.sh\n</code></pre> <p>\u5982\u679c\u662f\u90e8\u7f72 <code>containerd</code> \u7684\u7528\u6237\u53ef\u4ee5\u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\uff1a</p> <pre><code>ctr -n k8s.io run -d --net-host --privileged --mount=\"type=bind,src=/etc/ovn/,dst=/etc/ovn,options=rbind:rw\" --mount=\"type=bind,src=/var/run/ovn,dst=/var/run/ovn,options=rbind:rw\" --mount=\"type=bind,src=/var/log/ovn,dst=/var/log/ovn,options=rbind:rw\"  --env=\"NODE_IPS=\"192.168.65.3,192.168.65.2,192.168.65.1\"\" --env=\"LOCAL_IP=\"192.168.65.2\"\" --env=\"LEADER_IP=\"192.168.65.3\"\" docker.io/kubeovn/kube-ovn:v1.13.0 ovn-ic-db bash start-ic-db.sh\n</code></pre> <ul> <li><code>LOCAL_IP</code>\uff1a \u5f53\u524d\u5bb9\u5668\u6240\u5728\u8282\u70b9 IP \u5730\u5740\u3002</li> <li><code>NODE_IPS</code>\uff1a \u8fd0\u884c <code>OVN-IC</code> \u6570\u636e\u5e93\u7684\u4e09\u4e2a\u8282\u70b9 IP \u5730\u5740\uff0c\u4f7f\u7528\u9017\u53f7\u8fdb\u884c\u5206\u9694\u3002</li> <li><code>LEADER_IP</code>: \u8fd0\u884c <code>OVN-IC</code> \u6570\u636e\u5e93 leader \u8282\u70b9\u7684 IP \u5730\u5740\u3002</li> </ul> <p>\u5728\u6bcf\u4e2a\u96c6\u7fa4\u521b\u5efa <code>ovn-ic-config</code> \u65f6\u6307\u5b9a\u591a\u4e2a <code>OVN-IC</code> \u6570\u636e\u5e93\u8282\u70b9\u5730\u5740\uff1a</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: ovn-ic-config\nnamespace: kube-system\ndata:\nenable-ic: \"true\"\naz-name: \"az1\" ic-db-host: \"192.168.65.3,192.168.65.2,192.168.65.1\"\nic-nb-port: \"6645\"\nic-sb-port: \"6646\"\ngw-nodes: \"az1-gw\"\nauto-route: \"true\"\n</code></pre>"},{"location":"advance/with-ovn-ic/#_4","title":"\u624b\u52a8\u91cd\u7f6e","text":"<p>\u5728\u4e00\u4e9b\u60c5\u51b5\u4e0b\uff0c\u7531\u4e8e\u914d\u7f6e\u9519\u8bef\u9700\u8981\u5bf9\u6574\u4e2a\u4e92\u8054\u914d\u7f6e\u8fdb\u884c\u6e05\u7406\uff0c\u53ef\u4ee5\u53c2\u8003\u4e0b\u9762\u7684\u6b65\u9aa4\u6e05\u7406\u73af\u5883\u3002</p> <p>\u5220\u9664\u5f53\u524d\u7684 <code>ovn-ic-config</code> Configmap\uff1a</p> <pre><code>kubectl -n kube-system delete cm ovn-ic-config\n</code></pre> <p>\u5220\u9664 <code>ts</code> \u903b\u8f91\u4ea4\u6362\u673a\uff1a</p> <pre><code>kubectl-ko nbctl ls-del ts\n</code></pre> <p>\u5728\u5bf9\u7aef\u96c6\u7fa4\u91cd\u590d\u540c\u6837\u7684\u6b65\u9aa4\u3002</p>"},{"location":"advance/with-ovn-ic/#az-name","title":"\u4fee\u6539 az-name","text":"<p>\u53ef\u4ee5\u76f4\u63a5\u901a\u8fc7 <code>kubectl edit</code> \u7684\u65b9\u5f0f\u5bf9 <code>ovn-ic-config</code> \u8fd9\u4e2a configmap \u4e2d\u7684 <code>az-name</code> \u5b57\u6bb5\u8fdb\u884c\u4fee\u6539\u3002 \u4f46\u662f\u9700\u8981\u5728\u6bcf\u4e2a ovn-cni pod \u4e0a\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u5426\u5219\u53ef\u80fd\u51fa\u73b0\u6700\u957f 10 \u5206\u949f\u7684\u8de8\u96c6\u7fa4\u7f51\u7edc\u4e2d\u65ad\u3002</p> <pre><code>ovn-appctl -t ovn-controller inc-engine/recompute\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/with-submariner/","title":"\u4f7f\u7528 Submariner \u8fdb\u884c\u591a\u96c6\u7fa4\u4e92\u8054","text":"<p>Submariner \u4f5c\u4e3a\u53ef\u4ee5\u6253\u901a\u591a\u4e2a Kubernetes \u96c6\u7fa4 Pod \u548c Service \u7f51\u7edc\u7684\u5f00\u6e90\u7f51\u7edc\u7ec4\u4ef6\uff0c\u80fd\u591f\u5e2e\u52a9  Kube-OVN \u5b9e\u73b0\u591a\u96c6\u7fa4\u4e92\u8054\u3002</p> <p>\u76f8\u6bd4\u901a\u8fc7 OVN-IC \u6253\u901a\u591a\u96c6\u7fa4\u7f51\u7edc\u7684\u65b9\u5f0f\uff0cSubmariner \u53ef\u4ee5\u6253\u901a Kube-OVN \u548c\u975e Kube-OVN \u7684\u96c6\u7fa4\u7f51\u7edc\uff0c\u5e76 \u80fd\u63d0\u4f9b Service \u7684\u8de8\u96c6\u7fa4\u80fd\u529b\u3002\u4f46\u662f Submariner \u76ee\u524d\u53ea\u80fd\u5b9e\u73b0\u9ed8\u8ba4\u5b50\u7f51\u7684\u6253\u901a\uff0c\u65e0\u6cd5\u5b9e\u73b0\u591a\u5b50\u7f51\u9009\u62e9\u6027\u6253\u901a\u3002</p>"},{"location":"advance/with-submariner/#_1","title":"\u524d\u63d0\u6761\u4ef6","text":"<ul> <li>\u4e24\u4e2a\u96c6\u7fa4\u7684 Service CIDR \u548c\u9ed8\u8ba4\u5b50\u7f51\u7684 CIDR \u4e0d\u80fd\u91cd\u53e0\u3002</li> </ul>"},{"location":"advance/with-submariner/#submariner_1","title":"\u90e8\u7f72 Submariner","text":"<p>\u4e0b\u8f7d <code>subctl</code> \u4e8c\u8fdb\u5236\u6587\u4ef6\uff0c\u5e76\u90e8\u7f72\u5230\u76f8\u5e94\u8def\u5f84\uff1a</p> <pre><code>curl -Ls https://get.submariner.io | bash\nexport PATH=$PATH:~/.local/bin\necho export PATH=\\$PATH:~/.local/bin &gt;&gt; ~/.profile\n</code></pre> <p>\u5207\u6362 <code>kubeconfig</code> \u81f3\u5e0c\u671b\u90e8\u7f72 <code>submariner-broker</code> \u7684\u96c6\u7fa4\u8fdb\u884c\u90e8\u7f72\uff1a</p> <pre><code>subctl deploy-broker\n</code></pre> <p>\u5728\u672c\u6587\u6863\u4e2d <code>cluster0</code> \u7684\u9ed8\u8ba4\u5b50\u7f51 CIDR \u4e3a <code>10.16.0.0/16</code>\uff0cjoin \u5b50\u7f51 CIDR \u4e3a <code>100.64.0.0/16</code>\uff0c<code>cluster1</code> \u7684\u9ed8\u8ba4\u5b50\u7f51 CIDR \u4e3a <code>11.16.0.0/16</code>\uff0cjoin \u5b50\u7f51 CIDR \u4e3a '100.68.0.0/16'\u3002</p> <p>\u5207\u6362 <code>kubeconfig</code> \u81f3 <code>cluster0</code> \u6ce8\u518c\u96c6\u7fa4\u81f3 broker\uff0c\u5e76\u6ce8\u518c\u7f51\u5173\u8282\u70b9:</p> <pre><code>subctl  join broker-info.subm --clusterid  cluster0 --clustercidr 100.64.0.0/16,10.16.0.0/16  --natt=false --cable-driver vxlan --health-check=false\nkubectl label nodes cluster0 submariner.io/gateway=true\n</code></pre> <p>\u5207\u6362 <code>kubeconfig</code> \u81f3 <code>cluster1</code> \u6ce8\u518c\u96c6\u7fa4\u81f3 broker\uff0c\u5e76\u6ce8\u518c\u7f51\u5173\u8282\u70b9:</p> <pre><code>subctl  join broker-info.subm --clusterid  cluster1 --clustercidr 100.68.0.0/16,11.16.0.0/16  --natt=false --cable-driver vxlan --health-check=false\nkubectl label nodes cluster1 submariner.io/gateway=true\n</code></pre> <p>\u5982\u679c\u6267\u884c<code>join</code>\u547d\u4ee4\u4e4b\u540e\u6ca1\u6709\u65b0\u7684<code>gateway, routeagent</code>pod \u51fa\u73b0\u7684\u8bdd, \u8bf7\u4e3a<code>submariner-operator</code>\u8fd9\u4e2a<code>clusterrole</code>\u589e\u52a0\u4ee5\u4e0b\u6743\u9650:</p> <pre><code>- apiGroups:\n- \"apps\"\nresources:\n- daemonsets\nverbs:\n- create\n- get\n- list\n- watch\n- update\n</code></pre> <p>\u5bf9\u4e8e\u591a\u8282\u70b9\u7684\u96c6\u7fa4\uff0c\u9700\u8981\u5c06\u9ed8\u8ba4\u7684<code>subnet</code> <code>ovn-default</code>\u7684\u7f51\u5173\u914d\u7f6e\u6539\u4e3a<code>centralized</code>\u3002\u4e3a submariner \u914d\u7f6e\u7684<code>gateway</code>\u8282\u70b9\u9700\u8981\u548c<code>subnet</code>\u8282\u70b9\u5b8c\u5168\u76f8\u540c\u3002</p> <p>\u63a5\u4e0b\u6765\u53ef\u4ee5\u5728\u4e24\u4e2a\u96c6\u7fa4\u5185\u5206\u522b\u542f\u52a8 Pod \u5e76\u5c1d\u8bd5\u4f7f\u7528 IP \u8fdb\u884c\u76f8\u4e92\u8bbf\u95ee\u3002</p> <p>\u5982\u679c\u51fa\u73b0\u7f51\u7edc\u4e92\u901a\u95ee\u9898\u53ef\u901a\u8fc7 <code>subctl</code> \u547d\u4ee4\u8fdb\u884c\u8bca\u65ad\uff1a</p> <pre><code>subctl show all\nsubctl diagnose all\n</code></pre> <p>\u66f4\u591a Submariner \u76f8\u5173\u64cd\u4f5c\u8bf7\u67e5\u770b Submariner \u7528\u6237\u624b\u518c\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/custom-routes/","title":"\u81ea\u5b9a\u4e49\u8def\u7531","text":"<p>\u53ef\u4ee5\u5728\u521b\u5efa Pod \u65f6\u901a\u8fc7 Annotations \u6765\u6307\u5b9a\u9700\u8981\u914d\u7f6e\u7684\u8def\u7531\uff0c\u5982\u4e0b\u6240\u793a\uff1a</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: custom-routes\nannotations:\novn.kubernetes.io/routes: |\n[{\n\"dst\": \"192.168.0.101/24\",\n\"gw\": \"10.16.0.254\"\n}, {\n\"gw\": \"10.16.0.254\"\n}]'\nspec:\ncontainers:\n- name: nginx\nimage: docker.io/library/nginx:alpine\n</code></pre> <p><code>dst</code> \u5b57\u6bb5\u4e3a\u7a7a\u8868\u793a\u4fee\u6539\u9ed8\u8ba4\u8def\u7531\u3002</p> <p>\u5982\u679c\u5de5\u4f5c\u8d1f\u8f7d\u4e3a Deployment\u3001DaemonSet \u6216 StatefulSet\uff0c\u5bf9\u5e94\u7684 Annotation \u9700\u8981\u914d\u7f6e\u5728\u8d44\u6e90\u7684 <code>.spec.template.metadata.annotations</code> \u4e2d\uff0c\u793a\u4f8b\u5982\u4e0b\uff1a</p> <pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nname: custom-routes\nlabels:\napp: nginx\nspec:\nreplicas: 2\nselector:\nmatchLabels:\napp: nginx\ntemplate:\nmetadata:\nlabels:\napp: nginx\nannotations:\novn.kubernetes.io/routes: |\n[{\n\"dst\": \"192.168.0.101/24\",\n\"gw\": \"10.16.0.254\"\n}, {\n\"gw\": \"10.16.0.254\"\n}]'\nspec:\ncontainers:\n- name: nginx\nimage: docker.io/library/nginx:alpine\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/dual-stack/","title":"\u53cc\u6808\u4f7f\u7528","text":"<p>Kube-OVN \u4e2d\u4e0d\u540c\u7684\u5b50\u7f51\u53ef\u4ee5\u652f\u6301\u4e0d\u540c\u7684\u534f\u8bae\uff0c\u4e00\u4e2a\u96c6\u7fa4\u5185\u53ef\u4ee5\u540c\u65f6\u5b58\u5728 IPv4\uff0cIPv6 \u548c\u53cc\u6808\u7c7b\u578b\u7684\u5b50\u7f51\u3002 \u6211\u4eec\u63a8\u8350\u4e00\u4e2a\u96c6\u7fa4\u5185\u4f7f\u7528\u7edf\u4e00\u7684\u534f\u8bae\u7c7b\u578b\u4ee5\u7b80\u5316\u4f7f\u7528\u548c\u7ef4\u62a4\u3002</p> <p>\u4e3a\u4e86\u652f\u6301\u53cc\u6808\uff0c\u9700\u8981\u4e3b\u673a\u7f51\u7edc\u6ee1\u8db3\u53cc\u6808\u8981\u6c42\uff0c\u540c\u65f6\u9700\u8981\u5bf9 Kubernetes \u76f8\u5173\u53c2\u6570\u505a\u8c03\u6574\uff0c \u8bf7\u53c2\u8003 Kubernetes \u7684\u53cc\u6808\u5b98\u65b9\u6307\u5bfc\u3002</p>"},{"location":"guide/dual-stack/#_2","title":"\u521b\u5efa\u53cc\u6808\u5b50\u7f51","text":"<p>\u5728\u914d\u7f6e\u53cc\u6808\u65f6\uff0c\u53ea\u9700\u8981\u8bbe\u7f6e\u5bf9\u5e94\u5b50\u7f51 CIDR \u683c\u5f0f\u4e3a <code>cidr=&lt;IPv4 CIDR&gt;,&lt;IPv6 CIDR&gt;</code> \u5373\u53ef\u3002 CIDR \u987a\u5e8f\u8981\u6c42 IPv4 \u5728\u524d\uff0cIPv6 \u5728\u540e\uff0c\u5982\u4e0b\u6240\u793a\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata: name: ovn-test\nspec:\ncidrBlock: 10.16.0.0/16,fd00:10:16::/64\nexcludeIps:\n- 10.16.0.1\n- fd00:10:16::1\ngateway: 10.16.0.1,fd00:10:16::1\n</code></pre> <p>\u5982\u679c\u9700\u8981\u5728\u5b89\u88c5\u65f6\u9ed8\u8ba4\u5b50\u7f51\u4f7f\u7528\u53cc\u6808\uff0c\u9700\u8981\u5728\u5b89\u88c5\u811a\u672c\u4e2d\u4fee\u6539\u5982\u4e0b\u53c2\u6570\uff1a</p> <pre><code>POD_CIDR=\"10.16.0.0/16,fd00:10:16::/64\"\nJOIN_CIDR=\"100.64.0.0/16,fd00:100:64::/64\"\n</code></pre>"},{"location":"guide/dual-stack/#pod","title":"\u67e5\u770b Pod \u5730\u5740","text":"<p>\u914d\u7f6e\u53cc\u6808\u7f51\u7edc\u7684 Pod \u5c06\u4f1a\u4ece\u8be5\u5b50\u7f51\u540c\u65f6\u5206\u914d IPv4 \u548c IPv6 \u7684\u5730\u5740\uff0c\u5206\u914d\u7ed3\u679c\u4f1a\u663e\u793a\u5728 Pod \u7684 annotation \u4e2d:</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nannotations:\novn.kubernetes.io/allocated: \"true\"\novn.kubernetes.io/cidr: 10.16.0.0/16,fd00:10:16::/64\novn.kubernetes.io/gateway: 10.16.0.1,fd00:10:16::1\novn.kubernetes.io/ip_address: 10.16.0.9,fd00:10:16::9\novn.kubernetes.io/logical_switch: ovn-default\novn.kubernetes.io/mac_address: 00:00:00:14:88:09\novn.kubernetes.io/network_types: geneve\novn.kubernetes.io/routed: \"true\"\n...\npodIP: 10.16.0.9\npodIPs:\n- ip: 10.16.0.9\n- ip: fd00:10:16::9\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/eip-snat/","title":"EIP \u548c SNAT \u914d\u7f6e","text":"<p>\u8be5\u914d\u7f6e\u9488\u5bf9\u9ed8\u8ba4 VPC \u4e0b\u7684\u7f51\u7edc\uff0c\u7528\u6237\u81ea\u5b9a\u4e49 VPC \u8bf7\u53c2\u8003 VPC \u7f51\u5173</p> <p>Kube-OVN \u652f\u6301\u5229\u7528 OVN \u4e2d\u7684 L3 Gateway \u529f\u80fd\u6765\u5b9e\u73b0 Pod \u7ea7\u522b\u7684 SNAT \u548c EIP \u529f\u80fd\u3002 \u901a\u8fc7\u4f7f\u7528 SNAT\uff0c\u4e00\u7ec4 Pod \u53ef\u4ee5\u5171\u4eab\u4e00\u4e2a IP \u5730\u5740\u5bf9\u5916\u8fdb\u884c\u8bbf\u95ee\u3002 \u901a\u8fc7 EIP \u7684\u529f\u80fd\uff0c\u4e00\u4e2a Pod \u53ef\u4ee5\u76f4\u63a5\u548c\u4e00\u4e2a\u5916\u90e8 IP \u5173\u8054\uff0c \u5916\u90e8\u670d\u52a1\u53ef\u4ee5\u901a\u8fc7 EIP \u76f4\u63a5\u8bbf\u95ee Pod\uff0cPod \u4e5f\u5c06\u901a\u8fc7\u8fd9\u4e2a EIP \u8bbf\u95ee\u5916\u90e8\u670d\u52a1\u3002</p> <p></p>"},{"location":"guide/eip-snat/#_1","title":"\u51c6\u5907\u5de5\u4f5c","text":"<ul> <li>\u4e3a\u4e86\u4f7f\u7528 OVN \u7684 L3 Gateway \u80fd\u529b\uff0c\u5fc5\u987b\u5c06\u4e00\u4e2a\u5355\u72ec\u7684\u7f51\u5361\u63a5\u5165 OVS \u7f51\u6865\u4e2d\u8fdb\u884c Overlay \u548c Underlay \u7f51\u7edc\u7684\u6253\u901a\uff0c \u4e3b\u673a\u5fc5\u987b\u6709\u5176\u4ed6\u7684\u7f51\u5361\u7528\u4e8e\u8fd0\u7ef4\u7ba1\u7406\u3002</li> <li>\u7531\u4e8e\u7ecf\u8fc7 NAT \u540e\u7684\u6570\u636e\u5305\u4f1a\u76f4\u63a5\u8fdb\u5165 Underlay \u7f51\u7edc\uff0c\u5fc5\u987b\u786e\u8ba4\u5f53\u524d\u7684\u7f51\u7edc\u67b6\u6784\u4e0b\u6b64\u7c7b\u6570\u636e\u5305\u53ef\u4ee5\u5b89\u5168\u901a\u8fc7\u3002</li> <li>\u76ee\u524d EIP \u548c SNAT \u5730\u5740\u6ca1\u6709\u51b2\u7a81\u68c0\u6d4b\uff0c\u9700\u8981\u7ba1\u7406\u5458\u624b\u52a8\u5206\u914d\u907f\u514d\u5730\u5740\u51b2\u7a81\u3002</li> </ul>"},{"location":"guide/eip-snat/#_2","title":"\u521b\u5efa\u914d\u7f6e\u6587\u4ef6","text":"<p>\u5728 <code>kube-system</code> \u4e0b\u521b\u5efa ConfigMap <code>ovn-external-gw-config</code>\uff1a</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: ovn-external-gw-config\nnamespace: kube-system\ndata:\nenable-external-gw: \"true\"\nexternal-gw-nodes: \"kube-ovn-worker\"\nexternal-gw-nic: \"eth1\"\nexternal-gw-addr: \"172.56.0.1/16\"\nnic-ip: \"172.56.0.254/16\"\nnic-mac: \"16:52:f3:13:6a:25\"\n</code></pre> <ul> <li><code>enable-external-gw</code>: \u662f\u5426\u5f00\u542f SNAT \u548c EIP \u529f\u80fd\u3002</li> <li><code>type</code>: <code>centrailized</code> \u6216 <code>distributed</code>\uff0c \u9ed8\u8ba4\u4e3a <code>centralized</code> \u5982\u679c\u4f7f\u7528 <code>distributed</code>\uff0c\u5219\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u90fd\u9700\u8981\u6709\u540c\u540d\u7f51\u5361\u6765\u627f\u62c5\u7f51\u5173\u529f\u80fd\u3002</li> <li><code>external-gw-nodes</code>: <code>centralized</code> \u6a21\u5f0f\u4e0b\uff0c\u627f\u62c5\u7f51\u5173\u4f5c\u7528\u7684\u8282\u70b9\u540d\uff0c\u9017\u53f7\u5206\u9694\u3002</li> <li><code>external-gw-nic</code>: \u8282\u70b9\u4e0a\u627f\u62c5\u7f51\u5173\u4f5c\u7528\u7684\u7f51\u5361\u540d\u3002</li> <li><code>external-gw-addr</code>: \u7269\u7406\u7f51\u7edc\u7f51\u5173\u7684 IP \u548c\u63a9\u7801\u3002</li> <li><code>nic-ip</code>,<code>nic-mac</code>: \u5206\u914d\u7ed9\u903b\u8f91\u7f51\u5173\u7aef\u53e3\u7684 IP \u548c Mac\uff0c\u9700\u4e3a\u7269\u7406\u6bb5\u672a\u88ab\u5360\u7528\u7684 IP \u548c Mac\u3002</li> </ul>"},{"location":"guide/eip-snat/#ovn-ovs","title":"\u89c2\u5bdf OVN \u548c OVS \u72b6\u6001\u786e\u8ba4\u914d\u7f6e\u751f\u6548","text":"<p>\u68c0\u67e5 OVN-NB \u72b6\u6001, \u786e\u8ba4 <code>ovn-external</code> \u903b\u8f91\u4ea4\u6362\u673a\u5b58\u5728\uff0c\u5e76\u4e14 <code>ovn-cluster-ovn-external</code> \u903b\u8f91\u8def\u7531\u5668\u7aef\u53e3\u4e0a \u7ed1\u5b9a\u4e86\u6b63\u786e\u7684\u5730\u5740\u548c chassis\u3002</p> <pre><code># kubectl ko nbctl show\nswitch 3de4cea7-1a71-43f3-8b62-435a57ef16a6 (ovn-external)\nport ln-ovn-external\n        type: localnet\n        addresses: [\"unknown\"]\nport ovn-external-ovn-cluster\n        type: router\n        router-port: ovn-cluster-ovn-external\nrouter e1eb83ad-34be-4ed5-9a02-fcc8b1d357c4 (ovn-cluster)\nport ovn-cluster-ovn-external\n        mac: \"ac:1f:6b:2d:33:f1\"\nnetworks: [\"172.56.0.100/16\"]\ngateway chassis: [a5682814-2e2c-46dd-9c1c-6803ef0dab66]\n</code></pre> <p>\u68c0\u67e5 OVS \u72b6\u6001\uff0c\u786e\u8ba4\u76f8\u5e94\u7684\u7f51\u5361\u5df2\u7ecf\u6865\u63a5\u8fdb <code>br-external</code> \u7f51\u6865\uff1a</p> <pre><code># kubectl ko vsctl ${gateway node name} show\ne7d81150-7743-4d6e-9e6f-5c688232e130\n    Bridge br-external\n        Port br-external\n            Interface br-external\n                type: internal\n        Port eno2\n            Interface eno2\n        Port patch-ln-ovn-external-to-br-int\n            Interface patch-ln-ovn-external-to-br-int\n                type: patch\n                options: {peer=patch-br-int-to-ln-ovn-external}\n</code></pre>"},{"location":"guide/eip-snat/#pod-eip-snat","title":"Pod \u914d\u7f6e EIP \u548c SNAT","text":"<p>\u53ef\u901a\u8fc7\u5728 Pod \u4e0a\u589e\u52a0 <code>ovn.kubernetes.io/snat</code> \u6216 <code>ovn.kubernetes.io/eip</code> annotation \u6765\u5206\u522b\u914d\u7f6e SNAT \u548c EIP\uff1a</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: pod-gw\nannotations:\novn.kubernetes.io/snat: 172.56.0.200\nspec:\ncontainers:\n- name: snat-pod\nimage: docker.io/library/nginx:alpine\n---\napiVersion: v1\nkind: Pod\nmetadata:\nname: pod-gw\nannotations:\novn.kubernetes.io/eip: 172.56.0.233\nspec:\ncontainers:\n- name: eip-pod\nimage: docker.io/library/nginx:alpine\n</code></pre> <p>\u53ef\u901a\u8fc7 kubectl \u6216\u5176\u4ed6\u5de5\u5177\u52a8\u6001\u8c03\u6574 Pod \u6240\u914d\u7f6e\u7684 EIP \u6216 SNAT \u89c4\u5219\uff0c\u66f4\u6539\u65f6\u8bf7\u6ce8\u610f\u8981\u540c\u65f6\u5220\u9664 <code>ovn.kubernetes.io/routed</code> annotation \u89e6\u53d1\u8def\u7531\u7684\u53d8\u66f4\uff1a</p> <pre><code>kubectl annotate pod pod-gw ovn.kubernetes.io/eip=172.56.0.221 --overwrite\nkubectl annotate pod pod-gw ovn.kubernetes.io/routed-\n</code></pre> <p>\u5f53 EIP \u6216 SNAT \u89c4\u5219\u751f\u6548\u540e\uff0c<code>ovn.kubernetes.io/routed</code> annotation \u4f1a\u88ab\u91cd\u65b0\u6dfb\u52a0\u3002</p>"},{"location":"guide/eip-snat/#_3","title":"\u9ad8\u7ea7\u914d\u7f6e","text":"<p><code>kube-ovn-controller</code> \u7684\u90e8\u5206\u542f\u52a8\u53c2\u6570\u53ef\u5bf9 SNAT \u548c EIP \u529f\u80fd\u8fdb\u884c\u9ad8\u9636\u914d\u7f6e\uff1a</p> <ul> <li><code>--external-gateway-config-ns</code>: Configmap <code>ovn-external-gw-config</code> \u6240\u5c5e Namespace\uff0c \u9ed8\u8ba4\u4e3a <code>kube-system</code>\u3002</li> <li><code>--external-gateway-net</code>: \u7269\u7406\u7f51\u5361\u6240\u6865\u63a5\u7684\u7f51\u6865\u540d\uff0c\u9ed8\u8ba4\u4e3a <code>external</code>\u3002</li> <li><code>--external-gateway-vlanid</code>: \u7269\u7406\u7f51\u7edc Vlan Tag \u53f7\uff0c\u9ed8\u8ba4\u4e3a 0\uff0c \u5373\u4e0d\u4f7f\u7528 Vlan\u3002</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/ippool/","title":"IP \u6c60\u4f7f\u7528","text":"<p>IP \u6c60\uff08IPPool\uff09\u662f\u6bd4\u5b50\u7f51\uff08Subnet\uff09\u66f4\u7ec6\u529b\u5ea6\u7684 IPAM \u7ba1\u7406\u5355\u5143\u3002\u4f60\u53ef\u4ee5\u901a\u8fc7 IP \u6c60\u5c06\u5b50\u7f51\u7f51\u6bb5\u7ec6\u5206\u4e3a\u591a\u4e2a\u5355\u5143\uff0c\u6bcf\u4e2a\u5355\u5143\u7ed1\u5b9a\u4e00\u4e2a\u6216\u591a\u4e2a\u547d\u540d\u7a7a\u95f4\uff08Namespace\uff09\u3002</p>"},{"location":"guide/ippool/#_1","title":"\u4f7f\u7528\u65b9\u6cd5","text":"<p>\u4f7f\u7528\u793a\u4f8b\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: IPPool\nmetadata:\nname: pool-1\nspec:\nsubnet: ovn-default\nips:\n- \"10.16.0.201\"\n- \"10.16.0.210/30\"\n- \"10.16.0.220..10.16.0.230\"\nnamespaces:\n- ns-1\n</code></pre> <p>\u5b57\u6bb5\u8bf4\u660e\uff1a</p> \u540d\u79f0 \u7528\u9014 \u5907\u6ce8 subnet \u6307\u5b9a\u6240\u5c5e\u5b50\u7f51 \u5fc5\u586b ips \u6307\u5b9a\u5305\u542b\u7684 IP \u8303\u56f4 \u652f\u6301 \u3001 \u4ee5\u53ca .. \u4e09\u79cd\u683c\u5f0f\uff0c\u652f\u6301 IPv6\u3002 namespaces \u7ed1\u5b9a\u547d\u540d\u7a7a\u95f4 \u53ef\u9009"},{"location":"guide/ippool/#_2","title":"\u6ce8\u610f\u4e8b\u9879","text":"<ol> <li>\u4e3a\u4fdd\u8bc1\u4e0e Workload \u901a\u7528 IP Pool \u56fa\u5b9a\u5730\u5740 \u7684\u517c\u5bb9\u6027\uff0cIP \u6c60\u7684\u540d\u79f0\u4e0d\u80fd\u662f\u4e00\u4e2a IP \u5730\u5740\uff1b</li> <li>IP \u6c60\u7684 <code>.spec.ips</code> \u53ef\u6307\u5b9a\u8d85\u51fa\u5b50\u7f51\u8303\u56f4\u7684 IP \u5730\u5740\uff0c\u4f46\u5b9e\u9645\u6709\u6548\u7684 IP \u5730\u5740\u662f <code>.spec.ips</code> \u4e0e\u5b50\u7f51 CIDR \u7684\u4ea4\u96c6\uff1b</li> <li>\u540c\u4e00\u4e2a\u5b50\u7f51\u7684\u4e0d\u540c IP \u6c60\uff0c\u4e0d\u80fd\u5305\u542b\u76f8\u540c\u7684\uff08\u6709\u6548\uff09IP \u5730\u5740\uff1b</li> <li>IP \u6c60\u7684 <code>.spec.ips</code> \u53ef\u52a8\u6001\u4fee\u6539\uff1b</li> <li>IP \u6c60\u4f1a\u7ee7\u627f\u5b50\u7f51\u7684\u4fdd\u7559 IP\uff0c\u4ece IP \u6c60\u968f\u673a\u5206\u914d IP \u5730\u5740\u65f6\uff0c\u4f1a\u8df3\u8fc7\u5305\u542b\u5728 IP \u6c60\u4e2d\u7684\u4fdd\u7559 IP\uff1b</li> <li>\u4ece\u5b50\u7f51\u968f\u673a\u5206\u914d IP \u5730\u5740\u65f6\uff0c\u53ea\u4f1a\u4ece\u5b50\u7f51\u6240\u6709 IP \u6c60\u4ee5\u5916\u7684\u8303\u56f4\u5206\u914d\u3002</li> </ol> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/loadbalancer-service/","title":"LoadBalancer \u7c7b\u578b Service","text":"<p>Kube-OVN \u5df2\u7ecf\u652f\u6301\u4e86 VPC \u548c VPC \u7f51\u5173\u7684\u5b9e\u73b0\uff0c\u5177\u4f53\u914d\u7f6e\u53ef\u4ee5\u53c2\u8003 VPC \u914d\u7f6e\u3002</p> <p>\u7531\u4e8e VPC \u7f51\u5173\u7684\u4f7f\u7528\u6bd4\u8f83\u590d\u6742\uff0c\u57fa\u4e8e VPC \u7f51\u5173\u7684\u5b9e\u73b0\u505a\u4e86\u7b80\u5316\uff0c\u652f\u6301\u5728\u9ed8\u8ba4 VPC \u4e0b\u521b\u5efa <code>LoadBalancer</code> \u7c7b\u578b\u7684 Service\uff0c\u5b9e\u73b0\u901a\u8fc7 LoadBalancerIP \u6765\u8bbf\u95ee\u9ed8\u8ba4 VPC \u4e0b\u7684 Service\u3002</p> <p>\u9996\u5148\u786e\u8ba4\u73af\u5883\u4e0a\u6ee1\u8db3\u4ee5\u4e0b\u6761\u4ef6\uff1a</p> <ol> <li>\u5b89\u88c5\u4e86 <code>multus-cni</code> \u548c <code>macvlan cni</code>\u3002</li> <li>LoadBalancer Service \u7684\u652f\u6301\uff0c\u662f\u5bf9 VPC \u7f51\u5173\u4ee3\u7801\u8fdb\u884c\u7b80\u5316\u5b9e\u73b0\u7684\uff0c\u4ecd\u7136\u4f7f\u7528 <code>vpc-nat-gw</code> \u7684\u955c\u50cf\uff0c\u4f9d\u8d56 macvlan \u63d0\u4f9b\u591a\u7f51\u5361\u529f\u80fd\u652f\u6301\u3002</li> <li>\u76ee\u524d\u53ea\u652f\u6301\u5728<code>\u9ed8\u8ba4 VPC</code> \u914d\u7f6e\uff0c\u81ea\u5b9a\u4e49 VPC \u4e0b\u7684 LoadBalancer \u652f\u6301\u53ef\u4ee5\u53c2\u8003 VPC \u7684\u6587\u6863 VPC \u914d\u7f6e\u3002</li> </ol>"},{"location":"guide/loadbalancer-service/#vpc-loadbalancer-service","title":"\u9ed8\u8ba4 VPC LoadBalancer Service \u914d\u7f6e\u6b65\u9aa4","text":""},{"location":"guide/loadbalancer-service/#_1","title":"\u5f00\u542f\u7279\u6027\u5f00\u5173","text":"<p>\u4fee\u6539 kube-system namespace \u4e0b\u7684 deployment <code>kube-ovn-controller</code>\uff0c\u5728 <code>args</code> \u4e2d\u589e\u52a0\u53c2\u6570  <code>--enable-lb-svc=true</code>\uff0c\u5f00\u542f\u529f\u80fd\u5f00\u5173\uff0c\u8be5\u53c2\u6570\u9ed8\u8ba4\u4e3a false\u3002</p> <pre><code>containers:\n- args:\n- /kube-ovn/start-controller.sh\n- --default-cidr=10.16.0.0/16\n- --default-gateway=10.16.0.1\n- --default-gateway-check=true\n- --enable-lb-svc=true                  // \u53c2\u6570\u8bbe\u7f6e\u4e3a true\n</code></pre>"},{"location":"guide/loadbalancer-service/#networkattachmentdefinition-crd","title":"\u521b\u5efa NetworkAttachmentDefinition CRD \u8d44\u6e90","text":"<p>\u53c2\u8003\u4ee5\u4e0b yaml\uff0c\u521b\u5efa <code>net-attach-def</code> \u8d44\u6e90:</p> <pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: lb-svc-attachment\nnamespace: kube-system\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.0\",\n\"type\": \"macvlan\",\n\"master\": \"eth0\",                         //\u7269\u7406\u7f51\u5361\uff0c\u6839\u636e\u5b9e\u9645\u60c5\u51b5\u914d\u7f6e\n\"mode\": \"bridge\"\n}'\n</code></pre> <p>\u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u901a\u8fc7\u7269\u7406\u7f51\u5361 <code>eth0</code> \u6765\u5b9e\u73b0\u591a\u7f51\u5361\u529f\u80fd\uff0c\u5982\u679c\u9700\u8981\u4f7f\u7528\u5176\u4ed6\u7269\u7406\u7f51\u5361\uff0c\u53ef\u4ee5\u901a\u8fc7\u4fee\u6539 <code>master</code> \u53d6\u503c\uff0c\u6307\u5b9a\u4f7f\u7528\u7684\u7269\u7406\u7f51\u5361\u540d\u79f0\u3002</p>"},{"location":"guide/loadbalancer-service/#subnet","title":"\u521b\u5efa Subnet","text":"<p>\u521b\u5efa\u7684 Subnet\uff0c\u7528\u4e8e\u7ed9 LoadBalancer Service \u5206\u914d LoadBalancerIP\uff0c\u8be5\u5730\u5740\u6b63\u5e38\u60c5\u51b5\u4e0b\u5728\u96c6\u7fa4\u5916\u5e94\u8be5\u53ef\u4ee5\u8bbf\u95ee\u5230\u3002\u53ef\u4ee5\u914d\u7f6e Underlay Subnet \u7528\u4e8e\u5730\u5740\u5206\u914d\u3002</p> <p>\u53c2\u8003\u4ee5\u4e0b yaml\uff0c\u521b\u5efa\u65b0\u5b50\u7f51\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: attach-subnet\nspec:\nprotocol: IPv4\nprovider: lb-svc-attachment.kube-system    # provider \u683c\u5f0f\u56fa\u5b9a\uff0c\u7531\u4e0a\u4e00\u6b65\u521b\u5efa\u7684 net-attach-def \u8d44\u6e90\u7684 Name.Namespace \u7ec4\u6210\ncidrBlock: 172.18.0.0/16\ngateway: 172.18.0.1\nexcludeIps:\n- 172.18.0.0..172.18.0.10\n</code></pre> <p>Subnet \u4e2d <code>provider</code> \u53c2\u6570\u4ee5 <code>ovn</code> \u6216\u8005\u4ee5 <code>.ovn</code> \u4e3a\u540e\u7f00\u7ed3\u675f\uff0c\u8868\u793a\u8be5\u5b50\u7f51\u662f\u7531 Kube-OVN \u7ba1\u7406\u4f7f\u7528\uff0c\u9700\u8981\u5bf9\u5e94\u521b\u5efa <code>logical switch</code> \u8bb0\u5f55\u3002</p> <p><code>provider</code> \u975e <code>ovn</code> \u6216\u8005\u975e <code>.ovn</code> \u4e3a\u540e\u7f00\u7ed3\u675f\uff0c\u5219 Kube-OVN \u53ea\u63d0\u4f9b IPAM \u529f\u80fd\uff0c\u8bb0\u5f55 IP \u5730\u5740\u5206\u914d\u60c5\u51b5\uff0c\u4e0d\u5bf9\u5b50\u7f51\u505a\u4e1a\u52a1\u903b\u8f91\u5904\u7406\u3002</p>"},{"location":"guide/loadbalancer-service/#loadbalancer-service_1","title":"\u521b\u5efa LoadBalancer Service","text":"<p>\u53c2\u8003\u4ee5\u4e0b yaml\uff0c\u521b\u5efa LoadBalancer Service\uff1a</p> <pre><code>apiVersion: v1\nkind: Service\nmetadata:\nannotations:\nlb-svc-attachment.kube-system.kubernetes.io/logical_switch: attach-subnet   #\u53ef\u9009\novn.kubernetes.io/attachmentprovider: lb-svc-attachment.kube-system          #\u5fc5\u987b\nlabels:\napp: dynamic\nname: test-service\nnamespace: default\nspec:\nloadBalancerIP: 172.18.0.18                                                   #\u53ef\u9009\nports:\n- name: test\nprotocol: TCP\nport: 80\ntargetPort: 80\nselector:\napp: dynamic\nsessionAffinity: None\ntype: LoadBalancer\n</code></pre> <p>\u5728 yaml \u4e2d\uff0cannotation <code>ovn.kubernetes.io/attachmentprovider</code>  \u4e3a\u5fc5\u586b\u9879\uff0c\u53d6\u503c\u7531\u7b2c\u4e00\u6b65\u521b\u5efa\u7684 <code>net-attach-def</code> \u8d44\u6e90\u7684 Name.Namespace \u7ec4\u6210\u3002\u8be5 annotation \u7528\u4e8e\u5728\u521b\u5efa Pod \u65f6\uff0c\u67e5\u627e <code>net-attach-def</code> \u8d44\u6e90\u3002</p> <p>\u53ef\u4ee5\u901a\u8fc7 annotation \u6307\u5b9a\u591a\u7f51\u5361\u5730\u5740\u5206\u914d\u4f7f\u7528\u7684\u5b50\u7f51\u3002annotation key \u683c\u5f0f\u4e3a net-attach-def \u8d44\u6e90\u7684 <code>Name.Namespace.kubernetes.io/logical_switch</code>\u3002\u8be5\u914d\u7f6e\u4e3a<code>\u53ef\u9009</code>\u9009\u9879\uff0c\u5728\u6ca1\u6709\u6307\u5b9a LoadBalancerIP \u5730\u5740\u7684\u60c5\u51b5\u4e0b\uff0c\u5c06\u4ece\u8be5\u5b50\u7f51\u52a8\u6001\u5206\u914d\u5730\u5740\uff0c\u586b\u5145\u5230 LoadBalancerIP \u5b57\u6bb5\u3002</p> <p>\u5982\u679c\u9700\u8981\u9759\u6001\u914d\u7f6e LoadBalancerIP \u5730\u5740\uff0c\u53ef\u4ee5\u914d\u7f6e <code>spec.loadBalancerIP</code> \u5b57\u6bb5\uff0c\u8be5\u5730\u5740\u9700\u8981\u5728\u6307\u5b9a\u5b50\u7f51\u7684\u5730\u5740\u8303\u56f4\u5185\u3002</p> <p>\u5728\u6267\u884c yaml \u521b\u5efa Service \u540e\uff0c\u5728 Service \u540c Namespace \u4e0b\uff0c\u53ef\u4ee5\u770b\u5230 Pod \u542f\u52a8\u4fe1\u606f\uff1a</p> <pre><code># kubectl get pod\nNAME                                      READY   STATUS    RESTARTS   AGE\nlb-svc-test-service-6869d98dd8-cjvll      1/1     Running   0          107m\n# kubectl get svc\nNAME              TYPE           CLUSTER-IP       EXTERNAL-IP   PORT(S)        AGE\ntest-service      LoadBalancer   10.109.201.193   172.18.0.18   80:30056/TCP   107m\n</code></pre> <p>\u6307\u5b9a <code>service.spec.loadBalancerIP</code> \u53c2\u6570\u65f6\uff0c\u6700\u7ec8\u5c06\u8be5\u53c2\u6570\u8d4b\u503c\u7ed9 service external-ip \u5b57\u6bb5\u3002\u4e0d\u6307\u5b9a\u7684\u60c5\u51b5\u4e0b\uff0c\u8be5\u53c2\u6570\u4e3a\u968f\u673a\u5206\u914d\u503c\u3002</p> <p>\u67e5\u770b\u6d4b\u8bd5 Pod \u7684 yaml \u8f93\u51fa\uff0c\u5b58\u5728\u591a\u7f51\u5361\u5206\u914d\u7684\u5730\u5740\u4fe1\u606f\uff1a</p> <pre><code># kubectl get pod -o yaml lb-svc-test-service-6869d98dd8-cjvll\napiVersion: v1\nkind: Pod\nmetadata:\n  annotations:\n    k8s.v1.cni.cncf.io/network-status: |-\n      [{\n\"name\": \"kube-ovn\",\n          \"ips\": [\n\"10.16.0.2\"\n],\n          \"default\": true,\n          \"dns\": {}\n},{\n\"name\": \"default/test-service\",\n          \"interface\": \"net1\",\n          \"mac\": \"ba:85:f7:02:9f:42\",\n          \"dns\": {}\n}]\nk8s.v1.cni.cncf.io/networks: default/test-service\n    k8s.v1.cni.cncf.io/networks-status: |-\n      [{\n\"name\": \"kube-ovn\",\n          \"ips\": [\n\"10.16.0.2\"\n],\n          \"default\": true,\n          \"dns\": {}\n},{\n\"name\": \"default/test-service\",\n          \"interface\": \"net1\",\n          \"mac\": \"ba:85:f7:02:9f:42\",\n          \"dns\": {}\n}]\novn.kubernetes.io/allocated: \"true\"\novn.kubernetes.io/cidr: 10.16.0.0/16\n    ovn.kubernetes.io/gateway: 10.16.0.1\n    ovn.kubernetes.io/ip_address: 10.16.0.2\n    ovn.kubernetes.io/logical_router: ovn-cluster\n    ovn.kubernetes.io/logical_switch: ovn-default\n    ovn.kubernetes.io/mac_address: 00:00:00:45:F4:29\n    ovn.kubernetes.io/pod_nic_type: veth-pair\n    ovn.kubernetes.io/routed: \"true\"\ntest-service.default.kubernetes.io/allocated: \"true\"\ntest-service.default.kubernetes.io/cidr: 172.18.0.0/16\n    test-service.default.kubernetes.io/gateway: 172.18.0.1\n    test-service.default.kubernetes.io/ip_address: 172.18.0.18\n    test-service.default.kubernetes.io/logical_switch: attach-subnet\n    test-service.default.kubernetes.io/mac_address: 00:00:00:AF:AA:BF\n    test-service.default.kubernetes.io/pod_nic_type: veth-pair\n</code></pre> <p>\u67e5\u770b Service \u7684\u4fe1\u606f\uff1a</p> <pre><code># kubectl get svc -o yaml test-service\napiVersion: v1\nkind: Service\nmetadata:\n  annotations:\n    kubectl.kubernetes.io/last-applied-configuration: |\n{\"apiVersion\":\"v1\",\"kind\":\"Service\",\"metadata\":{\"annotations\":{\"test-service.default.kubernetes.io/logical_switch\":\"attach-subnet\"},\"labels\":{\"app\":\"dynamic\"},\"name\":\"test-service\",\"namespace\":\"default\"},\"spec\":{\"ports\":[{\"name\":\"test\",\"port\":80,\"protocol\":\"TCP\",\"targetPort\":80}],\"selector\":{\"app\":\"dynamic\"},\"sessionAffinity\":\"None\",\"type\":\"LoadBalancer\"}}\novn.kubernetes.io/vpc: ovn-cluster\n    test-service.default.kubernetes.io/logical_switch: attach-subnet\n  creationTimestamp: \"2022-06-15T09:01:58Z\"\nlabels:\n    app: dynamic\n  name: test-service\n  namespace: default\n  resourceVersion: \"38485\"\nuid: 161edee1-7f6e-40f5-9e09-5a52c44267d0\nspec:\n  allocateLoadBalancerNodePorts: true\nclusterIP: 10.109.201.193\n  clusterIPs:\n  - 10.109.201.193\n  externalTrafficPolicy: Cluster\n  internalTrafficPolicy: Cluster\n  ipFamilies:\n  - IPv4\n  ipFamilyPolicy: SingleStack\n  ports:\n  - name: test\nnodePort: 30056\nport: 80\nprotocol: TCP\n    targetPort: 80\nselector:\n    app: dynamic\n  sessionAffinity: None\n  type: LoadBalancer\nstatus:\n  loadBalancer:\n    ingress:\n    - ip: 172.18.0.18\n</code></pre>"},{"location":"guide/loadbalancer-service/#loadbalancerip","title":"\u6d4b\u8bd5 LoadBalancerIP \u8bbf\u95ee","text":"<p>\u53c2\u8003\u4ee5\u4e0b yaml, \u521b\u5efa\u6d4b\u8bd5 Pod\uff0c\u4f5c\u4e3a Service \u7684 Endpoints \u63d0\u4f9b\u670d\u52a1:</p> <pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nlabels:\napp: dynamic\nname: dynamic\nnamespace: default\nspec:\nreplicas: 2\nselector:\nmatchLabels:\napp: dynamic\nstrategy:\nrollingUpdate:\nmaxSurge: 25%\nmaxUnavailable: 25%\ntype: RollingUpdate\ntemplate:\nmetadata:\ncreationTimestamp: null\nlabels:\napp: dynamic\nspec:\ncontainers:\n- image: docker.io/library/nginx:alpine\nimagePullPolicy: IfNotPresent\nname: nginx\ndnsPolicy: ClusterFirst\nrestartPolicy: Always\n</code></pre> <p>\u6b63\u5e38\u60c5\u51b5\u4e0b\uff0c\u63d0\u4f9b\u7684\u5b50\u7f51\u5730\u5740\uff0c\u5728\u96c6\u7fa4\u5916\u5e94\u8be5\u53ef\u4ee5\u8bbf\u95ee\u5230\u3002\u4e3a\u4e86\u7b80\u5355\u9a8c\u8bc1\uff0c\u5728\u96c6\u7fa4\u5185\u8bbf\u95ee Service \u7684 <code>LoadBalancerIP:Port</code>\uff0c\u67e5\u770b\u662f\u5426\u6b63\u5e38\u8bbf\u95ee\u6210\u529f\u3002</p> <pre><code># curl 172.18.0.11:80\n&lt;html&gt;\n&lt;head&gt;\n        &lt;title&gt;Hello World!&lt;/title&gt;\n        &lt;link href='//fonts.googleapis.com/css?family=Open+Sans:400,700' rel='stylesheet' type='text/css'&gt;\n        &lt;style&gt;\n        body {\nbackground-color: white;\ntext-align: center;\npadding: 50px;\nfont-family: \"Open Sans\",\"Helvetica Neue\",Helvetica,Arial,sans-serif;\n}\n#logo {\nmargin-bottom: 40px;\n}\n&lt;/style&gt;\n&lt;/head&gt;\n&lt;body&gt;\n                &lt;h1&gt;Hello World!&lt;/h1&gt;\n                                &lt;h3&gt;Links found&lt;/h3&gt;\n        &lt;h3&gt;I am on  dynamic-7d8d7874f5-hsgc4&lt;/h3&gt;\n        &lt;h3&gt;Cookie                  =&lt;/h3&gt;\n                                        &lt;b&gt;KUBERNETES&lt;/b&gt; listening in 443 available at tcp://10.96.0.1:443&lt;br /&gt;\n                                                &lt;h3&gt;my name is hanhouchao!&lt;/h3&gt;\n                        &lt;h3&gt; RequestURI='/'&lt;/h3&gt;\n&lt;/body&gt;\n&lt;/html&gt;\n</code></pre> <p>\u8fdb\u5165 Service \u521b\u5efa\u7684 Pod\uff0c\u67e5\u770b\u7f51\u7edc\u7684\u4fe1\u606f</p> <pre><code># ip a\n4: net1@if62: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1500 qdisc noqueue state UP group default\n    link/ether ba:85:f7:02:9f:42 brd ff:ff:ff:ff:ff:ff link-netnsid 0\ninet 172.18.0.18/16 scope global net1\n       valid_lft forever preferred_lft forever\n    inet6 fe80::b885:f7ff:fe02:9f42/64 scope link\n       valid_lft forever preferred_lft forever\n36: eth0@if37: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1400 qdisc noqueue state UP group default\n    link/ether 00:00:00:45:f4:29 brd ff:ff:ff:ff:ff:ff link-netnsid 0\ninet 10.16.0.2/16 brd 10.16.255.255 scope global eth0\n       valid_lft forever preferred_lft forever\n    inet6 fe80::200:ff:fe45:f429/64 scope link\n       valid_lft forever preferred_lft forever\n\n# ip rule\n0: from all lookup local\n32764: from all iif eth0 lookup 100\n32765: from all iif net1 lookup 100\n32766: from all lookup main\n32767: from all lookup default\n\n# ip route show table 100\ndefault via 172.18.0.1 dev net1\n10.109.201.193 via 10.16.0.1 dev eth0\n172.18.0.0/16 dev net1 scope link\n\n# iptables -t nat -L -n -v\nChain PREROUTING (policy ACCEPT 0 packets, 0 bytes)\npkts bytes target     prot opt in     out     source               destination\n    0     0 DNAT       tcp  --  *      *       0.0.0.0/0            172.18.0.18          tcp dpt:80 to:10.109.201.193:80\n\nChain INPUT (policy ACCEPT 0 packets, 0 bytes)\npkts bytes target     prot opt in     out     source               destination\n\nChain OUTPUT (policy ACCEPT 0 packets, 0 bytes)\npkts bytes target     prot opt in     out     source               destination\n\nChain POSTROUTING (policy ACCEPT 0 packets, 0 bytes)\npkts bytes target     prot opt in     out     source               destination\n    0     0 MASQUERADE  all  --  *      *       0.0.0.0/0            10.109.201.193\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/mirror/","title":"\u6d41\u91cf\u955c\u50cf","text":"<p>\u6d41\u91cf\u955c\u50cf\u529f\u80fd\u53ef\u4ee5\u5c06\u8fdb\u51fa\u5bb9\u5668\u7f51\u7edc\u7684\u6570\u636e\u5305\u8fdb\u884c\u590d\u5236\u5230\u4e3b\u673a\u7684\u7279\u5b9a\u7f51\u5361\u3002\u7ba1\u7406\u5458\u6216\u5f00\u53d1\u8005 \u53ef\u4ee5\u901a\u8fc7\u76d1\u542c\u8fd9\u5757\u7f51\u5361\u83b7\u5f97\u5b8c\u6574\u7684\u5bb9\u5668\u7f51\u7edc\u6d41\u91cf\u6765\u8fdb\u4e00\u6b65\u8fdb\u884c\u5206\u6790\uff0c\u76d1\u63a7\uff0c\u5b89\u5168\u5ba1\u8ba1\u7b49\u64cd\u4f5c\u3002 \u4e5f\u53ef\u548c\u4f20\u7edf\u7684 NPM \u5bf9\u63a5\u83b7\u53d6\u66f4\u7ec6\u7c92\u5ea6\u7684\u6d41\u91cf\u76d1\u63a7\u3002</p> <p>\u6d41\u91cf\u955c\u50cf\u529f\u80fd\u4f1a\u5e26\u6765\u4e00\u5b9a\u7684\u6027\u80fd\u635f\u5931\uff0c\u6839\u636e CPU \u6027\u80fd\u4ee5\u53ca\u6d41\u91cf\u7684\u7279\u5f81\uff0c\u4f1a\u6709 5%~10% \u7684 \u989d\u5916 CPU \u6d88\u8017\u3002</p> <p></p>"},{"location":"guide/mirror/#_2","title":"\u5168\u5c40\u6d41\u91cf\u955c\u50cf\u914d\u7f6e","text":"<p>\u6d41\u91cf\u955c\u50cf\u529f\u80fd\u9ed8\u8ba4\u4e3a\u5173\u95ed\u72b6\u6001\uff0c\u5982\u679c\u9700\u8981\u5f00\u542f\u8bf7\u4fee\u6539 <code>kube-ovn-cni</code> DaemonSet \u7684\u542f\u52a8\u53c2\u6570\uff1a</p> <ul> <li><code>--enable-mirror=true</code>\uff1a \u662f\u5426\u5f00\u542f\u6d41\u91cf\u955c\u50cf\u3002</li> <li><code>--mirror-iface=mirror0</code>: \u6d41\u91cf\u955c\u50cf\u6240\u590d\u5236\u5230\u7684\u7f51\u5361\u540d\u3002\u8be5\u7f51\u5361\u53ef\u4e3a\u4e3b\u673a\u4e0a\u5df2\u5b58\u5728\u7684\u4e00\u5757\u7269\u7406\u7f51\u5361\uff0c \u6b64\u65f6\u8be5\u7f51\u5361\u4f1a\u88ab\u6865\u63a5\u8fdb br-int \u7f51\u6865\uff0c\u955c\u50cf\u6d41\u91cf\u4f1a\u76f4\u63a5\u63a5\u5165\u5e95\u5c42\u4ea4\u6362\u673a\u3002\u82e5\u7f51\u5361\u540d\u4e0d\u5b58\u5728\uff0cKube-OVN \u4f1a\u81ea\u52a8 \u521b\u5efa\u4e00\u5757\u540c\u540d\u7684\u865a\u62df\u7f51\u5361\uff0c\u7ba1\u7406\u5458\u6216\u5f00\u53d1\u8005\u53ef\u4ee5\u5728\u5bbf\u4e3b\u673a\u4e0a\u901a\u8fc7\u8be5\u7f51\u5361\u83b7\u53d6\u5f53\u524d\u8282\u70b9\u6240\u6709\u6d41\u91cf\u3002\u9ed8\u8ba4\u4e3a <code>mirror0</code>\u3002</li> </ul> <p>\u63a5\u4e0b\u6765\u53ef\u4ee5\u7528 tcpdump \u6216\u5176\u4ed6\u6d41\u91cf\u5206\u6790\u5de5\u5177\u76d1\u542c <code>mirror0</code> \u4e0a\u7684\u6d41\u91cf\uff1a</p> <pre><code>tcpdump -ni mirror0\n</code></pre>"},{"location":"guide/mirror/#pod","title":"Pod \u7ea7\u522b\u6d41\u91cf\u955c\u50cf\u914d\u7f6e","text":"<p>\u5982\u679c\u53ea\u9700\u5bf9\u90e8\u5206 Pod \u6d41\u91cf\u8fdb\u884c\u955c\u50cf\uff0c\u5219\u9700\u8981\u5173\u95ed\u5168\u5c40\u7684\u6d41\u91cf\u955c\u50cf\u529f\u80fd\uff0c\u7136\u540e\u5728\u7279\u5b9a Pod \u4e0a\u589e\u52a0 <code>ovn.kubernetes.io/mirror</code> annotation \u6765\u5f00\u542f Pod \u7ea7\u522b\u6d41\u91cf\u955c\u50cf\u3002</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: mirror-pod\nnamespace: ls1\nannotations:\novn.kubernetes.io/mirror: \"true\"\nspec:\ncontainers:\n- name: mirror-pod\nimage: docker.io/library/nginx:alpine\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/networkpolicy-log/","title":"NetworkPolicy \u65e5\u5fd7","text":"<p>NetworkPolicy \u4e3a Kubernetes \u63d0\u4f9b\u7684\u7f51\u7edc\u7b56\u7565\u63a5\u53e3\uff0cKube-OVN \u901a\u8fc7 OVN \u7684 ACL \u8fdb\u884c\u4e86\u5b9e\u73b0\u3002 \u4f7f\u7528\u4e86 NetworkPolicy \u540e\u5982\u679c\u51fa\u73b0\u7f51\u7edc\u4e0d\u901a\u7684\u60c5\u51b5\uff0c\u96be\u4ee5\u5224\u65ad\u662f\u7f51\u7edc\u6545\u969c\u95ee\u9898\u8fd8\u662f NetworkPolicy \u89c4\u5219\u8bbe\u7f6e\u95ee\u9898\u5bfc\u81f4\u7684\u7f51\u7edc\u4e2d\u65ad\u3002 Kube-OVN \u63d0\u4f9b\u4e86 NetworkPolicy \u65e5\u5fd7\u529f\u80fd\uff0c\u5e2e\u52a9\u7ba1\u7406\u5458\u5feb\u901f\u5b9a\u4f4d NetworkPolicy Drop \u89c4\u5219\u662f\u5426\u547d\u4e2d\uff0c\u5e76\u8bb0\u5f55\u6709\u54ea\u4e9b\u975e\u6cd5\u8bbf\u95ee\u3002</p> <p>NetworkPolicy \u65e5\u5fd7\u529f\u80fd\u4e00\u65e6\u5f00\u542f\uff0c\u5bf9\u6bcf\u4e2a\u547d\u4e2d Drop \u89c4\u5219\u7684\u6570\u636e\u5305\u90fd\u9700\u8981\u6253\u5370\u65e5\u5fd7\uff0c\u4f1a\u5e26\u6765\u989d\u5916\u6027\u80fd\u5f00\u9500\u3002 \u5728\u6076\u610f\u653b\u51fb\u4e0b\uff0c\u77ed\u65f6\u95f4\u5927\u91cf\u65e5\u5fd7\u53ef\u80fd\u4f1a\u8017\u5c3d CPU\u3002\u6211\u4eec\u5efa\u8bae\u5728\u751f\u4ea7\u73af\u5883\u9ed8\u8ba4\u5173\u95ed\u65e5\u5fd7\u529f\u80fd\uff0c\u5728\u9700\u8981\u6392\u67e5\u95ee\u9898\u65f6\uff0c\u52a8\u6001\u5f00\u542f\u65e5\u5fd7\u3002</p>"},{"location":"guide/networkpolicy-log/#networkpolicy_1","title":"\u5f00\u542f NetworkPolicy \u65e5\u5fd7","text":"<p>\u5728\u9700\u8981\u5f00\u542f\u65e5\u5fd7\u8bb0\u5f55\u7684 NetworkPolicy \u4e2d\u589e\u52a0 annotation <code>ovn.kubernetes.io/enable_log</code>\uff0c\u5982\u4e0b\u6240\u793a\uff1a</p> <pre><code>apiVersion: networking.k8s.io/v1\nkind: NetworkPolicy\nmetadata:\nname: default-deny-ingress\nnamespace: kube-system\nannotations:\novn.kubernetes.io/enable_log: \"true\"\nspec:\npodSelector: {}\npolicyTypes:\n- Ingress\n</code></pre> <p>\u63a5\u4e0b\u6765\u53ef\u4ee5\u5728\u5bf9\u5e94 Pod \u6240\u5728\u4e3b\u673a\u7684 <code>/var/log/ovn/ovn-controller.log</code> \u4e2d\u89c2\u5bdf\u5230\u88ab\u4e22\u5f03\u6570\u636e\u5305\u7684\u65e5\u5fd7\uff1a</p> <pre><code># tail -f /var/log/ovn/ovn-controller.log\n2022-07-20T05:55:03.229Z|00394|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: udp,vlan_tci=0x0000,dl_src=00:00:00:21:b7:d1,dl_dst=00:00:00:8d:0b:86,nw_src=10.16.0.10,nw_dst=10.16.0.7,nw_tos=0,nw_ecn=0,nw_ttl=63,tp_src=54343,tp_dst=53\n2022-07-20T05:55:06.229Z|00395|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: udp,vlan_tci=0x0000,dl_src=00:00:00:21:b7:d1,dl_dst=00:00:00:8d:0b:86,nw_src=10.16.0.9,nw_dst=10.16.0.7,nw_tos=0,nw_ecn=0,nw_ttl=63,tp_src=44187,tp_dst=53\n2022-07-20T05:55:08.230Z|00396|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: udp,vlan_tci=0x0000,dl_src=00:00:00:21:b7:d1,dl_dst=00:00:00:8d:0b:86,nw_src=10.16.0.10,nw_dst=10.16.0.7,nw_tos=0,nw_ecn=0,nw_ttl=63,tp_src=54274,tp_dst=53\n2022-07-20T05:55:11.231Z|00397|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: udp,vlan_tci=0x0000,dl_src=00:00:00:21:b7:d1,dl_dst=00:00:00:8d:0b:86,nw_src=10.16.0.9,nw_dst=10.16.0.7,nw_tos=0,nw_ecn=0,nw_ttl=63,tp_src=32778,tp_dst=53\n2022-07-20T05:55:11.231Z|00398|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: udp,vlan_tci=0x0000,dl_src=00:00:00:21:b7:d1,dl_dst=00:00:00:8d:0b:86,nw_src=10.16.0.9,nw_dst=10.16.0.7,nw_tos=0,nw_ecn=0,nw_ttl=63,tp_src=34188,tp_dst=53\n2022-07-20T05:55:13.231Z|00399|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: udp,vlan_tci=0x0000,dl_src=00:00:00:21:b7:d1,dl_dst=00:00:00:8d:0b:86,nw_src=10.16.0.10,nw_dst=10.16.0.7,nw_tos=0,nw_ecn=0,nw_ttl=63,tp_src=43290,tp_dst=53\n2022-07-20T05:55:22.096Z|00400|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: icmp,vlan_tci=0x0000,dl_src=00:00:00:6c:42:91,dl_dst=00:00:00:a5:d7:63,nw_src=10.16.0.9,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0\n2022-07-20T05:55:22.097Z|00401|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: icmp,vlan_tci=0x0000,dl_src=00:00:00:6c:42:91,dl_dst=00:00:00:a5:d7:63,nw_src=10.16.0.9,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0\n2022-07-20T05:55:22.098Z|00402|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: icmp,vlan_tci=0x0000,dl_src=00:00:00:6c:42:91,dl_dst=00:00:00:a5:d7:63,nw_src=10.16.0.9,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0\n</code></pre>"},{"location":"guide/networkpolicy-log/#networkpolicy_2","title":"\u5173\u95ed NetworkPolicy \u65e5\u5fd7","text":"<p>\u5c06\u5bf9\u5e94 NetworkPolicy \u4e2d\u7684 annotation <code>ovn.kubernetes.io/enable_log</code> \u8bbe\u7f6e\u4e3a <code>false</code> \u5373\u53ef\u5173\u95ed NetworkPolicy \u65e5\u5fd7\uff1a</p> <pre><code>kubectl annotate networkpolicy -n kube-system default-deny-ingress ovn.kubernetes.io/enable_log=false --overwrite\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/prometheus-grafana/","title":"\u914d\u7f6e\u76d1\u63a7\u548c\u9762\u677f","text":"<p>Kube-OVN \u53ef\u4ee5\u5c06\u7f51\u7edc\u63a7\u5236\u5e73\u9762\u4fe1\u606f\u4ee5\u53ca\u7f51\u7edc\u6570\u636e\u5e73\u9762\u8d28\u91cf\u4fe1\u606f\u6307\u6807\u4ee5 Prometheus \u6240\u652f\u6301\u7684\u683c\u5f0f\u5bf9\u5916\u8f93\u51fa\u3002</p> <p>\u6211\u4eec\u4f7f\u7528 kube-prometheus \u6240\u63d0\u4f9b\u7684 CRD \u6765\u5b9a\u4e49\u76f8\u5e94\u7684 Prometheus \u76d1\u63a7\u89c4\u5219\u3002 \u7528\u6237\u9700\u8981\u9884\u5148\u5b89\u88c5 kube-prometheus \u6765\u542f\u7528\u76f8\u5173\u7684 CRD\u3002Kube-OVN \u6240\u652f\u6301\u7684\u5168\u90e8\u76d1\u63a7\u6307\u6807\u8bf7\u53c2\u8003 Kube-OVN \u76d1\u63a7\u6307\u6807\u3002</p> <p>\u5982\u679c\u4f7f\u7528\u539f\u751f Prometheus \u8bf7\u53c2\u8003\u914d\u7f6e\u539f\u751f Prometheus \u83b7\u53d6\u76d1\u63a7\u6570\u636e \u8fdb\u884c\u914d\u7f6e\u3002</p>"},{"location":"guide/prometheus-grafana/#prometheus-monitor","title":"\u5b89\u88c5 Prometheus Monitor","text":"<p>Kube-OVN \u4f7f\u7528 Prometheus Monitor CRD \u6765\u7ba1\u7406\u76d1\u63a7\u8f93\u51fa\uff1a</p> <pre><code># \u7f51\u54af\u8d28\u91cf\u76f8\u5173\u76d1\u63a7\u6307\u6807\nkubectl apply -f https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/pinger-monitor.yaml\n# kube-ovn-controller \u76f8\u5173\u76d1\u63a7\u6307\u6807\nkubectl apply -f https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/controller-monitor.yaml\n# kube-ovn-cni \u76f8\u5173\u76d1\u63a7\u6307\u6807\nkubectl apply -f https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/cni-monitor.yaml\n# ovn \u76f8\u5173\u76d1\u63a7\u6307\u6807\nkubectl apply -f https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/ovn-monitor.yaml\n</code></pre> <p>Prometheus \u62c9\u53d6\u76d1\u63a7\u65f6\u95f4\u95f4\u9694\u9ed8\u8ba4\u4e3a 15s\uff0c\u5982\u679c\u9700\u8981\u8c03\u6574\u9700\u8981\u4fee\u6539 yaml \u4e2d\u7684 <code>interval</code> \u5b57\u6bb5\u3002</p>"},{"location":"guide/prometheus-grafana/#grafana","title":"\u52a0\u8f7d Grafana \u9762\u677f","text":"<p>Kube-OVN \u8fd8\u63d0\u4f9b\u4e86\u9884\u5148\u5b9a\u4e49\u597d\u7684 Grafana Dashboard \u5c55\u793a\u63a7\u5236\u5e73\u9762\u548c\u6570\u636e\u5e73\u9762\u76f8\u5173\u4fe1\u606f\u3002</p> <p>\u4e0b\u8f7d\u5bf9\u5e94 Dashboard \u6a21\u677f\uff1a</p> <pre><code># \u7f51\u7edc\u8d28\u91cf\u76f8\u5173\u9762\u677f\nwget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/pinger-grafana.json\n# kube-ovn-controller \u76f8\u5173\u9762\u677f\nwget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/controller-grafana.json\n# kube-ovn-cni \u76f8\u5173\u9762\u677f\nwget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/cni-grafana.json\n# ovn \u76f8\u5173\u9762\u677f\nwget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/ovn-grafana.json\n# ovs \u76f8\u5173\u9762\u677f\nwget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/ovs-grafana.json\n</code></pre> <p>\u5728 Grafana \u4e2d\u5bfc\u5165\u6a21\u677f\uff0c\u5e76\u5c06\u6570\u636e\u6e90\u8bbe\u7f6e\u4e3a\u5bf9\u5e94\u7684 Prometheus \u5373\u53ef\u770b\u5230\u5982\u4e0b Dashboard\uff1a</p> <p><code>kube-ovn-controller</code> \u8fd0\u884c\u72b6\u51b5\u76f8\u5173\u9762\u677f\uff1a</p> <p></p> <p><code>kube-ovn-pinger</code> \u7f51\u7edc\u8d28\u91cf\u76f8\u5173\u9762\u677f\uff1a</p> <p></p> <p><code>kube-ovn-cni</code> \u8fd0\u884c\u72b6\u51b5\u76f8\u5173\u9762\u677f\uff1a</p> <p></p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/prometheus/","title":"\u914d\u7f6e\u539f\u751f Prometheus \u83b7\u53d6\u76d1\u63a7\u6570\u636e","text":"<p>Kube-OVN \u63d0\u4f9b\u4e86\u4e30\u5bcc\u7684\u76d1\u63a7\u6570\u636e\uff0c\u7528\u4e8e OVN/OVS \u5065\u5eb7\u72b6\u6001\u68c0\u67e5\uff0c\u4ee5\u53ca\u5bb9\u5668\u7f51\u7edc\u548c\u4e3b\u673a\u7f51\u7edc\u7684\u8fde\u901a\u6027\u68c0\u67e5\u3002Kube-OVN \u914d\u7f6e\u4e86 ServiceMonitor\uff0c\u53ef\u4ee5\u7528\u4e8e Prometheus \u52a8\u6001\u83b7\u53d6\u76d1\u63a7\u6307\u6807\u3002</p> <p>\u5728\u67d0\u4e9b\u60c5\u51b5\u4e0b\uff0c\u53ea\u5b89\u88c5\u4e86 Prometheus Server\uff0c\u6ca1\u6709\u5b89\u88c5\u5176\u4ed6\u7684\u7ec4\u4ef6\uff0c\u53ef\u4ee5\u901a\u8fc7\u4fee\u6539 Prometheus \u7684\u914d\u7f6e\uff0c\u52a8\u6001\u83b7\u53d6\u96c6\u7fa4\u73af\u5883\u7684\u76d1\u63a7\u6570\u636e\u3002</p>"},{"location":"guide/prometheus/#prometheus_1","title":"Prometheus \u914d\u7f6e","text":"<p>\u4ee5\u4e0b\u7684\u914d\u7f6e\u6587\u6863\uff0c\u53c2\u8003\u81ea Prometheus \u670d\u52a1\u53d1\u73b0\u3002</p>"},{"location":"guide/prometheus/#_1","title":"\u6743\u9650\u914d\u7f6e","text":"<p>Prometheus \u90e8\u7f72\u5728\u96c6\u7fa4\u5185\uff0c\u9700\u8981\u901a\u8fc7 k8s apiserver \u6765\u8bbf\u95ee\u96c6\u7fa4\u5185\u7684\u8d44\u6e90\uff0c\u4ece\u800c\u5b9e\u73b0\u67e5\u8be2\u4e1a\u52a1\u7684\u76d1\u63a7\u6570\u636e\u3002</p> <p>\u53c2\u8003\u4ee5\u4e0b yaml\uff0c\u914d\u7f6e Prometheus \u9700\u8981\u7684\u6743\u9650\uff1a</p> <pre><code>apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\nname: prometheus\nrules:\n- apiGroups: [\"\"]\nresources:\n- nodes\n- nodes/proxy\n- services\n- endpoints\n- pods\nverbs: [\"get\", \"list\", \"watch\"]\n- apiGroups:\n- extensions\nresources:\n- ingresses\nverbs: [\"get\", \"list\", \"watch\"]\n- nonResourceURLs: [\"/metrics\"]\nverbs: [\"get\"]\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\nname: prometheus\nnamespace: default\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\nname: prometheus\nroleRef:\napiGroup: rbac.authorization.k8s.io\nkind: ClusterRole\nname: prometheus\nsubjects:\n- kind: ServiceAccount\nname: prometheus\nnamespace: default\n</code></pre>"},{"location":"guide/prometheus/#prometheus_2","title":"Prometheus \u914d\u7f6e\u6587\u4ef6","text":"<p>Prometheus \u7684\u542f\u52a8\uff0c\u4f9d\u8d56\u4e8e\u914d\u7f6e\u6587\u4ef6 prometheus.yml\uff0c\u53ef\u4ee5\u5c06\u8be5\u6587\u4ef6\u5185\u5bb9\u914d\u7f6e\u5728 ConfigMap \u5185\uff0c\u52a8\u6001\u6302\u8f7d\u5230 Pod \u4e2d\u3002</p> <p>\u53c2\u8003\u4ee5\u4e0b yaml\uff0c\u521b\u5efa Prometheus \u4f7f\u7528\u7684 ConfigMap \u6587\u4ef6\uff1a</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: prometheus-config\ndata:\nprometheus.yml: |-\nglobal:\nscrape_interval:     15s \nevaluation_interval: 15s\nscrape_configs:\n- job_name: 'prometheus'\nstatic_configs:\n- targets: ['localhost:9090']\n\n- job_name: 'kubernetes-nodes'\ntls_config:\nca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\nbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\nkubernetes_sd_configs:\n- role: node\n\n- job_name: 'kubernetes-service'\ntls_config:\nca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\nbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\nkubernetes_sd_configs:\n- role: service\n\n- job_name: 'kubernetes-endpoints'\ntls_config:\nca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\nbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\nkubernetes_sd_configs:\n- role: endpoints\n\n- job_name: 'kubernetes-ingress'\ntls_config:\nca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\nbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\nkubernetes_sd_configs:\n- role: ingress\n\n- job_name: 'kubernetes-pods'\ntls_config:\nca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\nbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\nkubernetes_sd_configs:\n- role: pod\n</code></pre> <p>Prometheus \u63d0\u4f9b\u4e86\u57fa\u4e8e\u89d2\u8272\u67e5\u8be2 Kubernetes \u8d44\u6e90\u76d1\u63a7\u7684\u64cd\u4f5c\uff0c\u5177\u4f53\u914d\u7f6e\u53ef\u4ee5\u67e5\u770b\u5b98\u65b9\u6587\u6863 kubernetes_sd_config\u3002</p> <p>\u5728 Kubernetes \u96c6\u7fa4\u4e2d\uff0cPrometheus \u652f\u6301\u67e5\u8be2\u76d1\u63a7\u6307\u6807\u7684\u89d2\u8272\u5305\u542b node\u3001service\u3001pod\u3001endpoints \u548c ingress\u3002\u5728 ConfigMap \u914d\u7f6e\u6587\u4ef6\u4e2d\u7ed9\u51fa\u4e86\u4ee5\u4e0a\u5168\u90e8\u8d44\u6e90\u7684\u76d1\u63a7\u67e5\u8be2\u914d\u7f6e\u793a\u4f8b\uff0c\u53ef\u4ee5\u6839\u636e\u9700\u8981\u9009\u62e9\u914d\u7f6e\u3002</p>"},{"location":"guide/prometheus/#prometheus_3","title":"Prometheus \u90e8\u7f72","text":"<p>\u53c2\u8003\u4ee5\u4e0b yaml \u6587\u4ef6\uff0c\u90e8\u7f72 Prometheus Server\uff1a</p> <pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nlabels:\napp: prometheus\nname: prometheus\nnamespace: default\nspec:\nreplicas: 1\nselector:\nmatchLabels:\napp: prometheus\nstrategy:\nrollingUpdate:\nmaxSurge: 25%\nmaxUnavailable: 25%\ntype: RollingUpdate\ntemplate:\nmetadata:\nlabels:\napp: prometheus\nspec:\nserviceAccountName: prometheus\nserviceAccount: prometheus\ncontainers:\n- image: docker.io/prom/prometheus:latest\nimagePullPolicy: IfNotPresent\nname: prometheus\ncommand:\n- \"/bin/prometheus\"\nargs:\n- \"--config.file=/etc/prometheus/prometheus.yml\"\nports:\n- containerPort: 9090\nprotocol: TCP\nvolumeMounts:\n- mountPath: \"/etc/prometheus\"\nname: prometheus-config\nvolumes:\n- name: prometheus-config\nconfigMap:\nname: prometheus-config\n</code></pre> <p>\u5728\u90e8\u7f72\u5b8c Prometheus \u4e4b\u540e\uff0c\u53c2\u8003\u4ee5\u4e0b yaml \u6587\u4ef6\uff0c\u90e8\u7f72 Prometheus Service\uff1a</p> <pre><code>kind: Service\napiVersion: v1\nmetadata:\nname: prometheus\nnamespace: default\nlabels:\nname: prometheus\nspec:\nports:\n- name: test\nprotocol: TCP\nport: 9090\ntargetPort: 9090\ntype: NodePort\nselector:\napp: prometheus\nsessionAffinity: None\n</code></pre> <p>\u5c06 Prometheus \u901a\u8fc7 NodePort \u66b4\u9732\u540e\uff0c\u5373\u53ef\u901a\u8fc7\u8282\u70b9\u6765\u8bbf\u95ee Prometheus\u3002</p>"},{"location":"guide/prometheus/#prometheus_4","title":"Prometheus \u76d1\u63a7\u6570\u636e\u9a8c\u8bc1","text":"<p>\u67e5\u770b\u73af\u5883\u4e0a Prometheus \u76f8\u5173\u7684\u4fe1\u606f\uff1a</p> <pre><code># kubectl get svc \nNAME         TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)          AGE\nkubernetes   ClusterIP   10.4.0.1       &lt;none&gt;        443/TCP          8d\nprometheus   NodePort    10.4.102.222   &lt;none&gt;        9090:32611/TCP   8d\n# kubectl get pod -o wide\nNAME                          READY   STATUS    RESTARTS   AGE    IP          NODE              NOMINATED NODE   READINESS GATES\nprometheus-7544b6b84d-v9m8s   1/1     Running   0          3d5h   10.3.0.7    192.168.137.219   &lt;none&gt;           &lt;none&gt;\n# kubectl get endpoints -o wide\nNAME         ENDPOINTS                                                        AGE\nkubernetes   192.168.136.228:6443,192.168.136.232:6443,192.168.137.219:6443   8d\nprometheus   10.3.0.7:9090                                                    8d\n</code></pre> <p>\u901a\u8fc7 NodePort \u8bbf\u95ee Prometheus\uff0c\u67e5\u770b Status/Service Discovery \u52a8\u6001\u67e5\u8be2\u5230\u7684\u6570\u636e\uff1a</p> <p></p> <p>\u53ef\u4ee5\u770b\u5230\u5f53\u524d\u53ef\u4ee5\u67e5\u8be2\u5230\u96c6\u7fa4\u4e0a\u5168\u90e8\u7684 Service \u6570\u636e\u4fe1\u606f\u3002</p>"},{"location":"guide/prometheus/#_2","title":"\u914d\u7f6e\u67e5\u8be2\u6307\u5b9a\u7684\u8d44\u6e90","text":"<p>\u4ee5\u4e0a\u7684 ConfigMap \u914d\u7f6e\u4e2d\uff0c\u6ca1\u6709\u6dfb\u52a0\u8fc7\u6ee4\u6761\u4ef6\uff0c\u67e5\u8be2\u4e86\u6240\u6709\u7684\u8d44\u6e90\u6570\u636e\u3002\u5982\u679c\u53ea\u9700\u8981\u67d0\u4e2a\u89d2\u8272\u7684\u8d44\u6e90\u6570\u636e\uff0c\u5219\u53ef\u4ee5\u6dfb\u52a0\u8fc7\u6ee4\u6761\u4ef6\u3002</p> <p>\u4ee5 Service \u4e3a\u4f8b\uff0c\u4fee\u6539 ConfigMap \u5185\u5bb9\uff0c\u53ea\u67e5\u8be2\u5173\u5fc3\u7684 Service \u76d1\u63a7\u6570\u636e\u3002</p> <pre><code>    - job_name: 'kubernetes-service'\ntls_config:\nca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\nbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\nkubernetes_sd_configs:\n- role: service\nrelabel_configs:\n- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]\naction: \"keep\"\nregex: \"true\"\n- action: labelmap\nregex: __meta_kubernetes_service_label_(.+)\n- source_labels: [__meta_kubernetes_namespace]\ntarget_label: kubernetes_namespace\n- source_labels: [__meta_kubernetes_service_name]\ntarget_label: kubernetes_service_name\n- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]\naction: replace\ntarget_label: __metrics_path__\nregex: \"(.+)\"\n</code></pre> <p>Service \u9ed8\u8ba4\u76d1\u63a7\u8def\u5f84\u4e3a /metrics\u3002\u5982\u679c Service \u63d0\u4f9b\u7684\u76d1\u63a7\u6307\u6807\u662f\u5176\u4ed6\u7684\u8def\u5f84\uff0c\u53ef\u4ee5\u901a\u8fc7\u7ed9 Service \u6dfb\u52a0 annotation <code>prometheus.io/path</code> \u6765\u6307\u5b9a\u91c7\u96c6\u8def\u5f84\u3002</p> <p>\u5e94\u7528\u4ee5\u4e0a yaml\uff0c\u66f4\u65b0 ConfigMap \u4fe1\u606f\uff0c\u91cd\u5efa Prometheus Pod\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002</p> <p>\u67e5\u770b kube-system Namespace \u4e0b\u7684 Service \u4fe1\u606f\uff1a</p> <pre><code># kubectl get svc -n kube-system\nNAME                  TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)                  AGE\nkube-dns              ClusterIP   10.4.0.10      &lt;none&gt;        53/UDP,53/TCP,9153/TCP   13d\nkube-ovn-cni          ClusterIP   10.4.228.60    &lt;none&gt;        10665/TCP                13d\nkube-ovn-controller   ClusterIP   10.4.172.213   &lt;none&gt;        10660/TCP                13d\nkube-ovn-monitor      ClusterIP   10.4.242.9     &lt;none&gt;        10661/TCP                13d\nkube-ovn-pinger       ClusterIP   10.4.122.52    &lt;none&gt;        8080/TCP                 13d\novn-nb                ClusterIP   10.4.80.213    &lt;none&gt;        6641/TCP                 13d\novn-northd            ClusterIP   10.4.126.234   &lt;none&gt;        6643/TCP                 13d\novn-sb                ClusterIP   10.4.216.249   &lt;none&gt;        6642/TCP                 13d\n</code></pre> <p>\u7ed9 Service \u6dfb\u52a0 annotation <code>prometheus.io/scrape=\"true\"</code>\uff1a</p> <pre><code># kubectl annotate svc -n kube-system kube-ovn-cni  prometheus.io/scrape=true\nservice/kube-ovn-cni annotated\n# kubectl annotate svc -n kube-system kube-ovn-controller  prometheus.io/scrape=true\nservice/kube-ovn-controller annotated\n# kubectl annotate svc -n kube-system kube-ovn-monitor  prometheus.io/scrape=true\nservice/kube-ovn-monitor annotated\n# kubectl annotate svc -n kube-system kube-ovn-pinger  prometheus.io/scrape=true\nservice/kube-ovn-pinger annotated\n</code></pre> <p>\u67e5\u770b\u914d\u7f6e\u540e\u7684 Service \u4fe1\u606f\uff1a</p> <pre><code># kubectl get svc -o yaml -n kube-system kube-ovn-controller\napiVersion: v1\nkind: Service\nmetadata:\n  annotations:\n    helm.sh/chart-version: v3.10.0-alpha.55\n    helm.sh/original-name: kube-ovn-controller\n    ovn.kubernetes.io/vpc: ovn-cluster\n    prometheus.io/scrape: \"true\"                        // \u6dfb\u52a0\u7684 annotation\n  labels:\n    app: kube-ovn-controller\n  name: kube-ovn-controller\n  namespace: kube-system\nspec:\n  clusterIP: 10.4.172.213\n  clusterIPs:\n  - 10.4.172.213\n  internalTrafficPolicy: Cluster\n  ipFamilies:\n  - IPv4\n  ipFamilyPolicy: SingleStack\n  ports:\n  - name: metrics\n    port: 10660\nprotocol: TCP\n    targetPort: 10660\nselector:\n    app: kube-ovn-controller\n  sessionAffinity: None\n  type: ClusterIP\nstatus:\n  loadBalancer: {}\n</code></pre> <p>\u67e5\u770b Prometheus Status Targets \u4fe1\u606f\uff0c\u53ef\u4ee5\u770b\u5230\u53ea\u6709\u6dfb\u52a0\u4e86 annotation \u7684 Service \u88ab\u8fc7\u6ee4\u51fa\u6765\uff1a </p> <p>\u66f4\u591a\u5173\u4e8e relabel \u6dfb\u52a0\u8fc7\u6ee4\u53c2\u6570\u7684\u4fe1\u606f\uff0c\u53ef\u4ee5\u53c2\u8003 Prometheus-Relabel\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/qos/","title":"\u5bb9\u5668\u7f51\u7edc QoS \u914d\u7f6e","text":"<p>Kube-OVN \u652f\u6301\u57fa\u4e8e\u5355\u4e2a Pod \u7684\u4e24\u79cd\u4e0d\u540c\u7c7b\u578b\u7684 QoS\uff1a</p> <ul> <li>\u6700\u5927\u5e26\u5bbd\u9650\u5236 QoS\u3002</li> <li><code>linux-netem</code>\uff0c\u6a21\u62df\u8bbe\u5907\u5e72\u6270\u4e22\u5305\u7b49\u7684 QoS\uff0c\u53ef\u7528\u4e8e\u6a21\u62df\u6d4b\u8bd5\u3002</li> </ul> <p>\u76ee\u524d\u53ea\u652f\u6301 Pod \u7ea7\u522b QoS \u4e0d\u652f\u6301 Namespace \u6216 Subnet \u7ea7\u522b\u7684 QoS \u9650\u5236\u3002</p>"},{"location":"guide/qos/#qos_1","title":"\u57fa\u4e8e\u6700\u5927\u5e26\u5bbd\u9650\u5236\u7684 QoS","text":"<p>\u8be5\u7c7b\u578b\u7684 QoS \u53ef\u4ee5\u901a\u8fc7 Pod annotation \u52a8\u6001\u8fdb\u884c\u914d\u7f6e\uff0c\u53ef\u4ee5\u5728\u4e0d\u4e2d\u65ad Pod \u8fd0\u884c\u7684\u60c5\u51b5\u4e0b\u8fdb\u884c\u8c03\u6574\u3002 \u5e26\u5bbd\u9650\u901f\u7684\u5355\u4f4d\u4e3a <code>Mbit/s</code>\u3002</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: qos\nnamespace: ls1\nannotations:\novn.kubernetes.io/ingress_rate: \"3\"\novn.kubernetes.io/egress_rate: \"1\"\nspec:\ncontainers:\n- name: qos\nimage: docker.io/library/nginx:alpine\n</code></pre> <p>\u4f7f\u7528 annotation \u52a8\u6001\u8c03\u6574 QoS\uff1a</p> <pre><code>kubectl annotate --overwrite  pod nginx-74d5899f46-d7qkn ovn.kubernetes.io/ingress_rate=3\n</code></pre>"},{"location":"guide/qos/#qos_2","title":"\u6d4b\u8bd5 QoS \u8c03\u6574","text":"<p>\u90e8\u7f72\u6027\u80fd\u6d4b\u8bd5\u9700\u8981\u7684\u5bb9\u5668\uff1a</p> <pre><code>kind: DaemonSet\napiVersion: apps/v1\nmetadata:\nname: perf\nnamespace: ls1\nlabels:\napp: perf\nspec:\nselector:\nmatchLabels:\napp: perf\ntemplate:\nmetadata:\nlabels:\napp: perf\nspec:\ncontainers:\n- name: nginx\nimage: docker.io/kubeovn/perf\n</code></pre> <p>\u8fdb\u5165\u5176\u4e2d\u4e00\u4e2a Pod \u5e76\u5f00\u542f iperf3 server\uff1a</p> <pre><code># kubectl exec -it perf-4n4gt -n ls1 sh\n# iperf3 -s\n-----------------------------------------------------------\nServer listening on 5201\n-----------------------------------------------------------\n</code></pre> <p>\u8fdb\u5165\u53e6\u4e00\u4e2a Pod \u8bf7\u6c42\u4e4b\u524d\u7684 Pod\uff1a</p> <pre><code># kubectl exec -it perf-d4mqc -n ls1 sh\n# iperf3 -c 10.66.0.12\nConnecting to host 10.66.0.12, port 5201\n[  4] local 10.66.0.14 port 51544 connected to 10.66.0.12 port 5201\n[ ID] Interval           Transfer     Bandwidth       Retr  Cwnd\n[  4]   0.00-1.00   sec  86.4 MBytes   725 Mbits/sec    3    350 KBytes\n[  4]   1.00-2.00   sec  89.9 MBytes   754 Mbits/sec  118    473 KBytes\n[  4]   2.00-3.00   sec   101 MBytes   848 Mbits/sec  184    586 KBytes\n[  4]   3.00-4.00   sec   104 MBytes   875 Mbits/sec  217    671 KBytes\n[  4]   4.00-5.00   sec   111 MBytes   935 Mbits/sec  175    772 KBytes\n[  4]   5.00-6.00   sec   100 MBytes   840 Mbits/sec  658    598 KBytes\n[  4]   6.00-7.00   sec   106 MBytes   890 Mbits/sec  742    668 KBytes\n[  4]   7.00-8.00   sec   102 MBytes   857 Mbits/sec  764    724 KBytes\n[  4]   8.00-9.00   sec  97.4 MBytes   817 Mbits/sec  1175    764 KBytes\n[  4]   9.00-10.00  sec   111 MBytes   934 Mbits/sec  1083    838 KBytes\n- - - - - - - - - - - - - - - - - - - - - - - - -\n[ ID] Interval           Transfer     Bandwidth       Retr\n[  4]   0.00-10.00  sec  1010 MBytes   848 Mbits/sec  5119             sender\n[  4]   0.00-10.00  sec  1008 MBytes   846 Mbits/sec                  receiver\n\niperf Done.\n</code></pre> <p>\u4fee\u6539\u7b2c\u4e00\u4e2a Pod \u7684\u5165\u53e3\u5e26\u5bbd QoS\uff1a</p> <pre><code>kubectl annotate --overwrite  pod perf-4n4gt -n ls1 ovn.kubernetes.io/ingress_rate=30\n</code></pre> <p>\u518d\u6b21\u4ece\u7b2c\u4e8c\u4e2a Pod \u6d4b\u8bd5\u7b2c\u4e00\u4e2a Pod \u5e26\u5bbd\uff1a</p> <pre><code># iperf3 -c 10.66.0.12\nConnecting to host 10.66.0.12, port 5201\n[  4] local 10.66.0.14 port 52372 connected to 10.66.0.12 port 5201\n[ ID] Interval           Transfer     Bandwidth       Retr  Cwnd\n[  4]   0.00-1.00   sec  3.66 MBytes  30.7 Mbits/sec    2   76.1 KBytes\n[  4]   1.00-2.00   sec  3.43 MBytes  28.8 Mbits/sec    0    104 KBytes\n[  4]   2.00-3.00   sec  3.50 MBytes  29.4 Mbits/sec    0    126 KBytes\n[  4]   3.00-4.00   sec  3.50 MBytes  29.3 Mbits/sec    0    144 KBytes\n[  4]   4.00-5.00   sec  3.43 MBytes  28.8 Mbits/sec    0    160 KBytes\n[  4]   5.00-6.00   sec  3.43 MBytes  28.8 Mbits/sec    0    175 KBytes\n[  4]   6.00-7.00   sec  3.50 MBytes  29.3 Mbits/sec    0    212 KBytes\n[  4]   7.00-8.00   sec  3.68 MBytes  30.9 Mbits/sec    0    294 KBytes\n[  4]   8.00-9.00   sec  3.74 MBytes  31.4 Mbits/sec    0    398 KBytes\n[  4]   9.00-10.00  sec  3.80 MBytes  31.9 Mbits/sec    0    526 KBytes\n- - - - - - - - - - - - - - - - - - - - - - - - -\n[ ID] Interval           Transfer     Bandwidth       Retr\n[  4]   0.00-10.00  sec  35.7 MBytes  29.9 Mbits/sec    2             sender\n[  4]   0.00-10.00  sec  34.5 MBytes  29.0 Mbits/sec                  receiver\n\niperf Done.\n</code></pre>"},{"location":"guide/qos/#linux-netem-qos","title":"linux-netem QoS","text":"<p>Pod \u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b annotation \u914d\u7f6e <code>linux-netem</code> \u7c7b\u578b QoS\uff1a <code>ovn.kubernetes.io/latency</code>\u3001<code>ovn.kubernetes.io/limit</code> \u548c <code>ovn.kubernetes.io/loss</code>\u3002</p> <ul> <li><code>ovn.kubernetes.io/latency</code>\uff1a\u8bbe\u7f6e Pod \u6d41\u91cf\u5ef6\u8fdf\uff0c\u53d6\u503c\u4e3a\u6574\u6570\uff0c\u5355\u4f4d\u4e3a ms\u3002</li> <li><code>ovn.kubernetes.io/limit</code>\uff1a \u4e3a <code>qdisc</code> \u961f\u5217\u53ef\u5bb9\u7eb3\u7684\u6700\u5927\u6570\u636e\u5305\u6570\uff0c\u53d6\u503c\u4e3a\u6574\u5f62\u6570\u503c\uff0c\u4f8b\u5982 1000\u3002</li> <li><code>ovn.kubernetes.io/loss</code>\uff1a \u4e3a\u8bbe\u7f6e\u7684\u62a5\u6587\u4e22\u5305\u6982\u7387\uff0c\u53d6\u503c\u4e3a float \u7c7b\u578b\uff0c\u4f8b\u5982\u53d6\u503c\u4e3a 20\uff0c\u5219\u4e3a\u8bbe\u7f6e 20% \u7684\u4e22\u5305\u6982\u7387\u3002</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/setup-options/","title":"\u5b89\u88c5\u548c\u914d\u7f6e\u9009\u9879","text":"<p>\u5728\u4e00\u952e\u5b89\u88c5\u4e2d\u6211\u4eec\u4f7f\u7528\u9ed8\u8ba4\u914d\u7f6e\u8fdb\u884c\u5b89\u88c5\uff0cKube-OVN \u8fd8\u652f\u6301\u66f4\u591a \u81ea\u5b9a\u4e49\u914d\u7f6e\uff0c\u53ef\u5728\u5b89\u88c5\u811a\u672c\u4e2d\u8fdb\u884c\u914d\u7f6e\uff0c\u6216\u8005\u4e4b\u540e\u66f4\u6539\u5404\u4e2a\u7ec4\u4ef6\u7684\u53c2\u6570\u6765\u8fdb\u884c\u914d\u7f6e\u3002\u672c\u6587\u6863\u5c06\u4f1a\u4ecb\u7ecd\u8fd9\u4e9b\u81ea\u5b9a\u4e49\u9009\u9879 \u7684\u4f5c\u7528\uff0c\u4ee5\u53ca\u5982\u4f55\u8fdb\u884c\u914d\u7f6e\u3002</p>"},{"location":"guide/setup-options/#_2","title":"\u5185\u7f6e\u7f51\u7edc\u8bbe\u7f6e","text":"<p>Kube-OVN \u5728\u5b89\u88c5\u65f6\u4f1a\u914d\u7f6e\u4e24\u4e2a\u5185\u7f6e\u5b50\u7f51\uff1a</p> <ol> <li><code>default</code> \u5b50\u7f51\uff0c\u4f5c\u4e3a Pod \u5206\u914d IP \u4f7f\u7528\u7684\u9ed8\u8ba4\u5b50\u7f51\uff0c\u9ed8\u8ba4 CIDR \u4e3a <code>10.16.0.0/16</code>\uff0c\u7f51\u5173\u4e3a <code>10.16.0.1</code>\u3002</li> <li><code>join</code> \u5b50\u7f51\uff0c\u4f5c\u4e3a Node \u548c Pod \u4e4b\u95f4\u8fdb\u884c\u7f51\u7edc\u901a\u4fe1\u7684\u7279\u6b8a\u5b50\u7f51, \u9ed8\u8ba4 CIDR \u4e3a <code>100.64.0.0/16</code>\uff0c\u7f51\u5173\u4e3a <code>100.64.0.1</code>\u3002</li> </ol> <p>\u5728\u5b89\u88c5\u65f6\u53ef\u4ee5\u901a\u8fc7\u5b89\u88c5\u811a\u672c\u5185\u7684\u914d\u7f6e\u8fdb\u884c\u66f4\u6539\uff1a</p> <pre><code>POD_CIDR=\"10.16.0.0/16\"\nPOD_GATEWAY=\"10.16.0.1\"\nJOIN_CIDR=\"100.64.0.0/16\"\nEXCLUDE_IPS=\"\"\n</code></pre> <p><code>EXCLUDE_IP</code> \u53ef\u8bbe\u7f6e <code>POD_CIDR</code> \u4e0d\u8fdb\u884c\u5206\u914d\u7684\u5730\u5740\u8303\u56f4\uff0c\u683c\u5f0f\u4e3a\uff1a<code>192.168.10.20..192.168.10.30</code>\u3002</p> <p>\u9700\u8981\u6ce8\u610f Overlay \u60c5\u51b5\u4e0b\u8fd9\u4e24\u4e2a\u7f51\u7edc\u4e0d\u80fd\u548c\u5df2\u6709\u7684\u4e3b\u673a\u7f51\u7edc\u548c Service CIDR \u51b2\u7a81\u3002</p> <p>\u5728\u5b89\u88c5\u540e\u53ef\u4ee5\u5bf9\u8fd9\u4e24\u4e2a\u7f51\u7edc\u7684\u5730\u5740\u8303\u56f4\u8fdb\u884c\u4fee\u6539\u8bf7\u53c2\u8003\u4fee\u6539\u9ed8\u8ba4\u5b50\u7f51\u548c\u4fee\u6539 Join \u5b50\u7f51\u3002</p>"},{"location":"guide/setup-options/#service","title":"Service \u7f51\u6bb5\u914d\u7f6e","text":"<p>\u7531\u4e8e\u90e8\u5206 <code>kube-proxy</code> \u8bbe\u7f6e\u7684 iptables \u548c\u8def\u7531\u89c4\u5219\u4f1a\u548c Kube-OVN \u8bbe\u7f6e\u7684\u89c4\u5219\u4ea7\u751f\u4ea4\u96c6\uff0c\u56e0\u6b64 Kube-OVN \u9700\u8981\u77e5\u9053 Service \u7684 CIDR \u6765\u6b63\u786e\u8bbe\u7f6e\u5bf9\u5e94\u7684\u89c4\u5219\u3002</p> <p>\u5728\u5b89\u88c5\u811a\u672c\u4e2d\u53ef\u4ee5\u901a\u8fc7\u4fee\u6539\uff1a</p> <pre><code>SVC_CIDR=\"10.96.0.0/12\"  </code></pre> <p>\u6765\u8fdb\u884c\u914d\u7f6e\u3002</p> <p>\u4e5f\u53ef\u4ee5\u5728\u5b89\u88c5\u540e\u901a\u8fc7\u4fee\u6539 <code>kube-ovn-controller</code> Deployment \u7684\u53c2\u6570\uff1a</p> <pre><code>args:\n- --service-cluster-ip-range=10.96.0.0/12\n</code></pre> <p>\u6765\u8fdb\u884c\u4fee\u6539\u3002</p>"},{"location":"guide/setup-options/#overlay","title":"Overlay \u7f51\u5361\u9009\u62e9","text":"<p>\u5728\u8282\u70b9\u5b58\u5728\u591a\u5757\u7f51\u5361\u7684\u60c5\u51b5\u4e0b\uff0cKube-OVN \u9ed8\u8ba4\u4f1a\u9009\u62e9 Kubernetes Node IP \u5bf9\u5e94\u7684\u7f51\u5361\u4f5c\u4e3a\u5bb9\u5668\u95f4\u8de8\u8282\u70b9\u901a\u4fe1\u7684\u7f51\u5361\u5e76\u5efa\u7acb\u5bf9\u5e94\u7684\u96a7\u9053\u3002</p> <p>\u5982\u679c\u9700\u8981\u9009\u62e9\u5176\u4ed6\u7684\u7f51\u5361\u5efa\u7acb\u5bb9\u5668\u96a7\u9053\uff0c\u53ef\u4ee5\u5728\u5b89\u88c5\u811a\u672c\u4e2d\u4fee\u6539\uff1a</p> <pre><code>IFACE=eth1\n</code></pre> <p>\u8be5\u9009\u9879\u652f\u6301\u4ee5\u9017\u53f7\u6240\u5206\u9694\u6b63\u5219\u8868\u8fbe\u5f0f,\u4f8b\u5982<code>ens[a-z0-9]*,eth[a-z0-9]*</code>\u3002</p> <p>\u5b89\u88c5\u540e\u4e5f\u53ef\u901a\u8fc7\u4fee\u6539 <code>kube-ovn-cni</code> DaemonSet \u7684\u53c2\u6570\u8fdb\u884c\u8c03\u6574\uff1a</p> <pre><code>args:\n- --iface=eth1\n</code></pre> <p>\u5982\u679c\u6bcf\u53f0\u673a\u5668\u7684\u7f51\u5361\u540d\u5747\u4e0d\u540c\uff0c\u4e14\u6ca1\u6709\u56fa\u5b9a\u89c4\u5f8b\uff0c\u53ef\u4ee5\u4f7f\u7528\u8282\u70b9 annotation <code>ovn.kubernetes.io/tunnel_interface</code> \u8fdb\u884c\u6bcf\u4e2a\u8282\u70b9\u7684\u9010\u4e00\u914d\u7f6e\uff0c\u62e5\u6709\u8be5 annotation \u8282\u70b9\u4f1a\u8986\u76d6 <code>iface</code> \u7684\u914d\u7f6e\uff0c\u4f18\u5148\u4f7f\u7528 annotation\u3002</p> <pre><code>kubectl annotate node no1 ovn.kubernetes.io/tunnel_interface=ethx\n</code></pre>"},{"location":"guide/setup-options/#mtu","title":"MTU \u8bbe\u7f6e","text":"<p>\u7531\u4e8e Overlay \u5c01\u88c5\u9700\u8981\u5360\u636e\u989d\u5916\u7684\u7a7a\u95f4\uff0cKube-OVN \u5728\u521b\u5efa\u5bb9\u5668\u7f51\u5361\u65f6\u4f1a\u6839\u636e\u9009\u62e9\u7f51\u5361\u7684 MTU \u8fdb\u884c\u5bb9\u5668\u7f51\u5361\u7684 MTU \u8c03\u6574\uff0c \u9ed8\u8ba4\u60c5\u51b5\u4e0b Overlay \u5b50\u7f51\u4e0b Pod \u7f51\u5361 MTU \u4e3a\u4e3b\u673a\u7f51\u5361 MTU - 100\uff0cUnderlay \u5b50\u7f51\u4e0b\uff0cPod \u7f51\u5361\u548c\u4e3b\u673a\u7f51\u5361\u6709\u76f8\u540c MTU\u3002</p> <p>\u5982\u679c\u9700\u8981\u8c03\u6574 Overlay \u5b50\u7f51\u4e0b MTU \u7684\u5927\u5c0f\uff0c\u53ef\u4ee5\u4fee\u6539 <code>kube-ovn-cni</code> DaemonSet \u7684\u53c2\u6570\uff1a</p> <pre><code>args:\n- --mtu=1333\n</code></pre>"},{"location":"guide/setup-options/#_3","title":"\u5168\u5c40\u6d41\u91cf\u955c\u50cf\u5f00\u542f\u8bbe\u7f6e","text":"<p>\u5728\u5f00\u542f\u5168\u5c40\u6d41\u91cf\u955c\u50cf\u7684\u60c5\u51b5\u4e0b\uff0cKube-OVN \u4f1a\u5728\u6bcf\u4e2a\u8282\u70b9\u4e0a\u521b\u5efa\u4e00\u5757 <code>mirror0</code> \u7684\u865a\u62df\u7f51\u5361\uff0c\u590d\u5236\u5f53\u524d\u673a\u5668\u6240\u6709\u5bb9\u5668\u7f51\u7edc\u6d41\u91cf\u5230\u8be5\u7f51\u5361\u4e0a\uff0c \u7528\u6237\u53ef\u4ee5\u901a\u8fc7 tcpdump \u53ca\u5176\u4ed6\u5de5\u5177\u8fdb\u884c\u6d41\u91cf\u5206\u6790\uff0c\u8be5\u529f\u80fd\u53ef\u4ee5\u5728\u5b89\u88c5\u811a\u672c\u4e2d\u901a\u8fc7\u4e0b\u9762\u7684\u914d\u7f6e\u5f00\u542f\uff1a</p> <pre><code>ENABLE_MIRROR=true\n</code></pre> <p>\u4e5f\u53ef\u5728\u5b89\u88c5\u540e\u901a\u8fc7\u4fee\u6539 <code>kube-ovn-cni</code> DaemonSet \u7684\u53c2\u6570\u65b9\u5f0f\u8fdb\u884c\u8c03\u6574:</p> <pre><code>args:\n- --enable-mirror=true\n</code></pre> <p>\u6d41\u91cf\u955c\u50cf\u7684\u80fd\u529b\u5728\u9ed8\u8ba4\u5b89\u88c5\u4e2d\u4e3a\u5173\u95ed\uff0c\u5982\u679c\u9700\u8981\u7ec6\u7c92\u5ea6\u7684\u6d41\u91cf\u955c\u50cf\u6216\u9700\u8981\u5c06\u6d41\u91cf\u955c\u50cf\u5230\u989d\u5916\u7684\u7f51\u5361\u8bf7\u53c2\u8003\u5bb9\u5668\u7f51\u7edc\u6d41\u91cf\u955c\u50cf\u3002</p>"},{"location":"guide/setup-options/#lb","title":"LB \u5f00\u542f\u8bbe\u7f6e","text":"<p>Kube-OVN \u4f7f\u7528 OVN \u4e2d\u7684 L2 LB \u6765\u5b9e\u73b0 Service \u8f6c\u53d1\uff0c\u5728 Overlay \u573a\u666f\u4e2d\uff0c\u7528\u6237\u53ef\u4ee5\u9009\u62e9\u4f7f\u7528 <code>kube-proxy</code> \u6765\u5b8c\u6210 Service \u6d41\u91cf\u8f6c\u53d1, \u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\u53ef\u4ee5\u5173\u95ed Kube-OVN \u7684 LB \u529f\u80fd\u4ee5\u8fbe\u5230\u63a7\u5236\u9762\u548c\u6570\u636e\u9762\u66f4\u597d\u7684\u6027\u80fd\u3002</p> <p>\u8be5\u529f\u80fd\u53ef\u4ee5\u5728\u5b89\u88c5\u811a\u672c\u4e2d\u8fdb\u884c\u914d\u7f6e\uff1a</p> <pre><code>ENABLE_LB=false\n</code></pre> <p>\u6216\u8005\u5728\u5b89\u88c5\u540e\u901a\u8fc7\u66f4\u6539 <code>kube-ovn-controller</code> Deployment \u7684\u53c2\u6570\u8fdb\u884c\u914d\u7f6e\uff1a</p> <pre><code>args:\n- --enable-lb=false\n</code></pre> <p>LB \u7684\u529f\u80fd\u5728\u9ed8\u8ba4\u5b89\u88c5\u4e2d\u4e3a\u5f00\u542f\u3002</p> <p>\u4ece Kube-OVN v1.12.0 \u7248\u672c\u5f00\u59cb\uff0c\u5728 subnet crd \u5b9a\u4e49\u4e2d\u589e\u52a0\u4e86 spec \u5b57\u6bb5 <code>enableLb</code>\uff0c\u5c06 Kube-OVN \u7684 LB \u529f\u80fd\u8fc1\u79fb\u5230\u5b50\u7f51\u5c42\u7ea7\uff0c\u53ef\u4ee5\u57fa\u4e8e\u4e0d\u540c\u7684\u5b50\u7f51\u5206\u522b\u8bbe\u7f6e\u662f\u5426\u5f00\u542f LB \u529f\u80fd\u3002<code>kube-ovn-controller</code> Deployment \u4e2d\u7684 <code>enable-lb</code> \u53c2\u6570\u4f5c\u4e3a\u5168\u5c40\u53c2\u6570\uff0c\u63a7\u5236\u662f\u5426\u521b\u5efa load-balancer \u8bb0\u5f55\uff0c\u5b50\u7f51\u4e2d\u65b0\u589e\u7684 <code>enableLb</code> \u53c2\u6570\u7528\u4e8e\u63a7\u5236\u5b50\u7f51\u662f\u5426\u5173\u8054 load-balancer \u8bb0\u5f55\u3002\u4e4b\u524d\u7248\u672c\u5347\u7ea7\u5230 v1.12.0 \u4e4b\u540e\uff0c\u5b50\u7f51 <code>enableLb</code> \u53c2\u6570\u4f1a\u81ea\u52a8\u7ee7\u627f\u539f\u6709\u7684\u5168\u5c40\u5f00\u5173\u53c2\u6570\u53d6\u503c\u3002</p>"},{"location":"guide/setup-options/#networkpolicy","title":"NetworkPolicy \u5f00\u542f\u8bbe\u7f6e","text":"<p>Kube-OVN \u4f7f\u7528 OVN \u4e2d\u7684 ACL \u6765\u5b9e\u73b0 NetworkPolicy\uff0c\u7528\u6237\u53ef\u4ee5\u9009\u62e9\u5173\u95ed NetworkPolicy \u529f\u80fd \u6216\u8005\u4f7f\u7528 Cilium Chain \u7684\u65b9\u5f0f\u5229\u7528 eBPF \u5b9e\u73b0 NetworkPolicy\uff0c \u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\u53ef\u4ee5\u5173\u95ed Kube-OVN \u7684 NetworkPolicy \u529f\u80fd\u4ee5\u8fbe\u5230\u63a7\u5236\u9762\u548c\u6570\u636e\u9762\u66f4\u597d\u7684\u6027\u80fd\u3002</p> <p>\u8be5\u529f\u80fd\u53ef\u4ee5\u5728\u5b89\u88c5\u811a\u672c\u4e2d\u8fdb\u884c\u914d\u7f6e\uff1a</p> <pre><code>ENABLE_NP=false\n</code></pre> <p>\u6216\u8005\u5728\u5b89\u88c5\u540e\u901a\u8fc7\u66f4\u6539 <code>kube-ovn-controller</code> Deployment \u7684\u53c2\u6570\u8fdb\u884c\u914d\u7f6e\uff1a</p> <pre><code>args:\n- --enable-np=false\n</code></pre> <p>NetworkPolicy \u7684\u80fd\u529b\u5728\u9ed8\u8ba4\u5b89\u88c5\u4e2d\u4e3a\u5f00\u542f\u3002</p>"},{"location":"guide/setup-options/#eip-snat","title":"EIP \u548c SNAT \u5f00\u542f\u8bbe\u7f6e","text":"<p>\u9ed8\u8ba4\u7f51\u7edc\u4e0b\u5982\u679c\u65e0\u9700\u4f7f\u7528 EIP \u548c SNAT \u7684\u80fd\u529b\uff0c\u53ef\u4ee5\u9009\u62e9\u5173\u95ed\u76f8\u5173\u529f\u80fd\uff0c\u4ee5\u51cf\u5c11 <code>kube-ovn-controller</code> \u5728\u521b\u5efa\u548c\u66f4\u65b0 \u7f51\u7edc\u65f6\u7684\u68c0\u67e5\u6d88\u8017\uff0c\u5728\u5927\u89c4\u6a21\u96c6\u7fa4\u73af\u5883\u4e0b\u53ef\u4ee5\u63d0\u5347\u5904\u7406\u901f\u5ea6\u3002</p> <p>\u8be5\u529f\u80fd\u53ef\u5728\u5b89\u88c5\u811a\u672c\u4e2d\u8fdb\u884c\u914d\u7f6e\uff1a</p> <pre><code>ENABLE_EIP_SNAT=false\n</code></pre> <p>\u6216\u8005\u5728\u5b89\u88c5\u540e\u901a\u8fc7\u66f4\u6539 <code>kube-ovn-controller</code> Deployment \u7684\u53c2\u6570\u8fdb\u884c\u914d\u7f6e\uff1a</p> <pre><code>args:\n- --enable-eip-snat=false\n</code></pre> <p>EIP \u548c SNAT \u7684\u80fd\u529b\u5728\u9ed8\u8ba4\u5b89\u88c5\u4e2d\u4e3a\u5f00\u542f\u3002\u8be5\u529f\u80fd\u7684\u76f8\u5173\u4f7f\u7528\u548c\u5176\u4ed6\u53ef\u914d\u53c2\u6570\u8bf7\u53c2\u8003 EIP \u548c SNAT \u914d\u7f6e\u3002</p>"},{"location":"guide/setup-options/#load-balancer-service","title":"Load Balancer \u7c7b\u578b Service \u652f\u6301\u5f00\u542f\u8bbe\u7f6e","text":"<p>\u9ed8\u8ba4 VPC \u4e0b\u53ef\u901a\u8fc7\u5f00\u542f\u8be5\u9009\u9879\u6765\u652f\u6301 Load Balancer \u7c7b\u578b Service\u3002\u8be5\u529f\u80fd\u7684\u76f8\u5173\u4f7f\u7528\u548c\u5176\u4ed6\u53ef\u914d\u53c2\u6570\u8bf7\u53c2\u8003 LoadBalancer \u7c7b\u578b Service\u3002</p> <p>\u8be5\u529f\u80fd\u9ed8\u8ba4\u5173\u95ed\uff0c\u53ef\u5728\u5b89\u88c5\u811a\u672c\u4e2d\u8fdb\u884c\u914d\u7f6e\uff1a</p> <pre><code>ENABLE_LB_SVC=true\n</code></pre> <p>\u6216\u8005\u5728\u5b89\u88c5\u540e\u901a\u8fc7\u66f4\u6539 <code>kube-ovn-controller</code> Deployment \u7684\u53c2\u6570\u8fdb\u884c\u914d\u7f6e\uff1a</p> <pre><code>args:\n- --enable-lb-svc=true\n</code></pre>"},{"location":"guide/setup-options/#ecmp","title":"\u96c6\u4e2d\u5f0f\u7f51\u5173 ECMP \u5f00\u542f\u8bbe\u7f6e","text":"<p>\u96c6\u4e2d\u5f0f\u7f51\u5173\u652f\u6301\u4e3b\u5907\u548c ECMP \u4e24\u79cd\u9ad8\u53ef\u7528\u6a21\u5f0f\uff0c\u5982\u679c\u9700\u8981\u542f\u7528 ECMP \u6a21\u5f0f\uff0c \u9700\u8981\u66f4\u6539 <code>kube-ovn-controller</code> Deployment \u7684\u53c2\u6570\u8fdb\u884c\u914d\u7f6e:</p> <pre><code>args:\n- --enable-ecmp=true </code></pre> <p>\u4ece Kube-OVN v1.12.0 \u7248\u672c\u5f00\u59cb\uff0c\u5728 subnet crd \u5b9a\u4e49\u4e2d\u589e\u52a0\u4e86 spec \u5b57\u6bb5 <code>enableEcmp</code>\uff0c\u5c06\u96c6\u4e2d\u5f0f\u5b50\u7f51 ECMP \u5f00\u5173\u63a7\u5236\u8fc1\u79fb\u5230\u5b50\u7f51\u5c42\u7ea7\uff0c\u53ef\u4ee5\u57fa\u4e8e\u4e0d\u540c\u7684\u5b50\u7f51\u5206\u522b\u8bbe\u7f6e\u662f\u5426\u5f00\u542f ECMP \u6a21\u5f0f\u3002\u539f\u6709\u7684 <code>kube-ovn-controller</code> Deployment \u4e2d\u7684 <code>enable-ecmp</code> \u53c2\u6570\u4e0d\u518d\u4f7f\u7528\u3002\u4e4b\u524d\u7248\u672c\u5347\u7ea7\u5230 v1.12.0 \u4e4b\u540e\uff0c\u5b50\u7f51\u5f00\u5173\u4f1a\u81ea\u52a8\u7ee7\u627f\u539f\u6709\u7684\u5168\u5c40\u5f00\u5173\u53c2\u6570\u53d6\u503c\u3002</p> <p>\u96c6\u4e2d\u5f0f\u7f51\u5173\u9ed8\u8ba4\u5b89\u88c5\u4e0b\u4e3a\u4e3b\u5907\u6a21\u5f0f\uff0c\u66f4\u591a\u7f51\u5173\u76f8\u5173\u5185\u5bb9\u8bf7\u53c2\u8003\u5b50\u7f51\u4f7f\u7528\u3002</p>"},{"location":"guide/setup-options/#kubevirt-vm","title":"Kubevirt VM \u56fa\u5b9a\u5730\u5740\u5f00\u542f\u8bbe\u7f6e","text":"<p>\u9488\u5bf9 Kubevirt \u521b\u5efa\u7684 VM \u5b9e\u4f8b\uff0c<code>kube-ovn-controller</code> \u53ef\u4ee5\u6309\u7167\u7c7b\u4f3c StatefulSet Pod \u7684\u65b9\u5f0f\u8fdb\u884c IP \u5730\u5740\u5206\u914d\u548c\u7ba1\u7406\u3002 \u4ee5\u8fbe\u5230 VM \u5b9e\u4f8b\u5728\u751f\u547d\u5468\u671f\u5185\u542f\u505c\uff0c\u5347\u7ea7\uff0c\u8fc1\u79fb\u7b49\u64cd\u4f5c\u8fc7\u7a0b\u4e2d\u5730\u5740\u56fa\u5b9a\u4e0d\u53d8\uff0c\u66f4\u7b26\u865a\u62df\u5316\u5408\u7528\u6237\u7684\u5b9e\u9645\u4f7f\u7528\u4f53\u9a8c\u3002</p> <p>\u8be5\u529f\u80fd\u5728 1.10.6 \u540e\u9ed8\u8ba4\u5f00\u542f\uff0c\u82e5\u8981\u5173\u95ed\u6b64\u529f\u80fd\uff0c\u9700\u8981\u5728 <code>kube-ovn-controller</code> Deployment \u7684\u542f\u52a8\u547d\u4ee4\u4e2d\u8bbe\u7f6e\u5982\u4e0b\u53c2\u6570\uff1a</p> <pre><code>args:\n- --keep-vm-ip=false\n</code></pre>"},{"location":"guide/setup-options/#cni","title":"CNI \u914d\u7f6e\u76f8\u5173\u8bbe\u7f6e","text":"<p>Kube-OVN \u9ed8\u8ba4\u4f1a\u5728 <code>/opt/cni/bin</code> \u76ee\u5f55\u4e0b\u5b89\u88c5 CNI \u6267\u884c\u6587\u4ef6\uff0c\u5728 <code>/etc/cni/net.d</code> \u76ee\u5f55\u4e0b\u5b89\u88c5 CNI \u914d\u7f6e\u6587\u4ef6 <code>01-kube-ovn.conflist</code>\u3002 \u5982\u679c\u9700\u8981\u66f4\u6539\u5b89\u88c5\u4f4d\u7f6e\u548c CNI \u914d\u7f6e\u6587\u4ef6\u7684\u4f18\u5148\u7ea7\uff0c\u53ef\u4ee5\u901a\u8fc7\u5b89\u88c5\u811a\u672c\u7684\u4e0b\u5217\u53c2\u6570\u8fdb\u884c\u8c03\u6574\uff1a</p> <pre><code>CNI_CONF_DIR=\"/etc/cni/net.d\"\nCNI_BIN_DIR=\"/opt/cni/bin\"\nCNI_CONFIG_PRIORITY=\"01\"\n</code></pre> <p>\u6216\u8005\u5728\u5b89\u88c5\u540e\u66f4\u6539 <code>kube-ovn-cni</code> DaemonSet \u7684 Volume \u6302\u8f7d\u548c\u542f\u52a8\u53c2\u6570\uff1a</p> <pre><code>volumes:\n- name: cni-conf\nhostPath:\npath: \"/etc/cni/net.d\"\n- name: cni-bin\nhostPath:\npath:\"/opt/cni/bin\"\n...\nargs:\n- --cni-conf-name=01-kube-ovn.conflist\n</code></pre>"},{"location":"guide/setup-options/#_4","title":"\u96a7\u9053\u7c7b\u578b\u8bbe\u7f6e","text":"<p>Kube-OVN \u9ed8\u8ba4 Overlay \u7684\u5c01\u88c5\u6a21\u5f0f\u4e3a Geneve\uff0c\u5982\u679c\u60f3\u66f4\u6362\u4e3a Vxlan \u6216 STT\uff0c\u53ef\u4ee5\u901a\u8fc7\u5b89\u88c5\u811a\u672c\u7684\u4e0b\u5217\u53c2\u6570\u8fdb\u884c\u8c03\u6574\uff1a</p> <pre><code>TUNNEL_TYPE=\"vxlan\"\n</code></pre> <p>\u6216\u8005\u5728\u5b89\u88c5\u540e\u66f4\u6539 <code>ovs-ovn</code> DaemonSet \u7684\u73af\u5883\u53d8\u91cf\uff1a</p> <pre><code>env:\n- name: TUNNEL_TYPE\nvalue: \"vxlan\"\n</code></pre> <p>\u5982\u679c\u9700\u8981\u4f7f\u7528 STT \u96a7\u9053\u9700\u8981\u989d\u5916\u7f16\u8bd1 ovs \u7684\u5185\u6838\u6a21\u5757\uff0c\u8bf7\u53c2\u8003\u6027\u80fd\u8c03\u4f18\u3002</p> <p>\u4e0d\u540c\u534f\u8bae\u5728\u5b9e\u9645\u4f7f\u7528\u4e2d\u7684\u533a\u522b\u8bf7\u53c2\u8003\u96a7\u9053\u534f\u8bae\u8bf4\u660e\u3002</p>"},{"location":"guide/setup-options/#ssl","title":"SSL \u8bbe\u7f6e","text":"<p>OVN DB \u7684 API \u63a5\u53e3\u652f\u6301 SSL \u52a0\u5bc6\u6765\u4fdd\u8bc1\u8fde\u63a5\u5b89\u5168\uff0c\u5982\u8981\u5f00\u542f\u53ef\u8c03\u6574\u5b89\u88c5\u811a\u672c\u4e2d\u7684\u5982\u4e0b\u53c2\u6570:</p> <pre><code>ENABLE_SSL=true\n</code></pre> <p>SSL \u529f\u80fd\u9ed8\u8ba4\u5b89\u88c5\u4e0b\u4e3a\u5173\u95ed\u6a21\u5f0f\u3002</p>"},{"location":"guide/setup-options/#ip","title":"\u7ed1\u5b9a\u672c\u5730 ip","text":"<p>kube-ovn-controller/kube-ovn-cni/kube-ovn-monitor \u8fd9\u4e9b\u670d\u52a1\u652f\u6301\u7ed1\u5b9a\u672c\u5730 ip\uff0c\u8be5\u529f\u80fd\u8bbe\u8ba1\u539f\u56e0\u4e3b\u8981\u662f\u56e0\u4e3a\u67d0\u4e9b\u573a\u666f\u4e0b\u51fa\u4e8e\u5b89\u5168\u8003\u8651\u4e0d\u5141\u8bb8\u670d\u52a1\u7ed1\u5b9a 0.0.0.0 \uff08\u6bd4\u5982\u8be5\u670d\u52a1\u90e8\u7f72\u5728\u67d0\u4e2a\u5bf9\u5916\u7f51\u5173\u4e0a\uff0c\u5916\u90e8\u7528\u6237\u53ef\u4ee5\u76f4\u63a5\u901a\u8fc7\u516c\u7f51 ip \u5e76\u6307\u5b9a\u7aef\u53e3\u53bb\u8bbf\u95ee\u5230\u8be5\u670d\u52a1\uff09\uff0c\u8be5\u529f\u80fd\u9ed8\u8ba4\u662f\u6253\u5f00\u7684\uff0c\u7531\u5b89\u88c5\u811a\u672c\u4e2d\u5982\u4e0b\u53c2\u6570\u63a7\u5236\uff1a</p> <pre><code>ENABLE_BIND_LOCAL_IP=true\n</code></pre> <p>\u4ee5 kube-ovn-monitor \u4e3a\u4f8b\uff0c\u5f00\u542f\u529f\u80fd\u540e\u4f1a\u628a\u670d\u52a1\u7ed1\u5b9a\u672c\u5730\u7684 pod ip \u5982\u4e0b\uff1a</p> <pre><code># netstat -tunlp |grep kube-ovn\ntcp        0      0 172.18.0.5:10661        0.0.0.0:*               LISTEN      2612/./kube-ovn-mon\n</code></pre> <p>\u5b89\u88c5\u540e\u4e5f\u53ef\u901a\u8fc7\u4fee\u6539\u670d\u52a1\u7684 deployment \u6216\u8005 daemonSet \u7684\u73af\u5883\u53d8\u91cf\u53c2\u6570\u8fdb\u884c\u8c03\u6574\uff1a</p> <pre><code>env:\n- name: ENABLE_BIND_LOCAL_IP\nvalue: \"false\"\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/static-ip-mac/","title":"\u56fa\u5b9a\u5730\u5740","text":"<p>Kube-OVN \u9ed8\u8ba4\u4f1a\u6839\u636e Pod \u6240\u5728 Namespace \u6240\u5c5e\u7684\u5b50\u7f51\u4e2d\u968f\u673a\u5206\u914d IP \u548c Mac\u3002 \u9488\u5bf9\u5de5\u4f5c\u8d1f\u8f7d\u9700\u8981\u56fa\u5b9a\u5730\u5740\u7684\u60c5\u51b5\uff0cKube-OVN \u6839\u636e\u4e0d\u540c\u7684\u573a\u666f\uff0c\u63d0\u4f9b\u4e86\u591a\u79cd\u56fa\u5b9a\u5730\u5740\u7684\u65b9\u6cd5\uff1a</p> <ul> <li>\u5355\u4e2a Pod \u56fa\u5b9a IP/Mac\u3002</li> <li>Workload \u901a\u7528 IP Pool \u65b9\u5f0f\u6307\u5b9a\u56fa\u5b9a\u5730\u5740\u8303\u56f4\u3002</li> <li>StatefulSet \u56fa\u5b9a\u5730\u5740\u3002</li> <li>KubeVirt VM \u56fa\u5b9a\u5730\u5740\u3002</li> </ul>"},{"location":"guide/static-ip-mac/#pod-ip-mac","title":"\u5355\u4e2a Pod \u56fa\u5b9a IP \u548c Mac","text":"<p>\u53ef\u4ee5\u5728\u521b\u5efa Pod \u65f6\u901a\u8fc7 annotation \u6765\u6307\u5b9a Pod \u8fd0\u884c\u65f6\u6240\u9700\u7684 IP/Mac, <code>kube-ovn-controller</code> \u8fd0\u884c\u65f6\u5c06\u4f1a\u8df3\u8fc7\u5730\u5740\u968f\u673a\u5206\u914d\u9636\u6bb5\uff0c\u7ecf\u8fc7\u51b2\u7a81\u68c0\u6d4b\u540e\u76f4\u63a5\u4f7f\u7528\u6307\u5b9a\u5730\u5740\uff0c\u5982\u4e0b\u6240\u793a\uff1a</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: static-ip\nannotations:\novn.kubernetes.io/ip_address: 10.16.0.15   // \u53cc\u6808\u5730\u5740\u4f7f\u7528\u9017\u53f7\u5206\u9694 10.16.0.15,fd00:10:16::15\novn.kubernetes.io/mac_address: 00:00:00:53:6B:B6\nspec:\ncontainers:\n- name: static-ip\nimage: docker.io/library/nginx:alpine\n</code></pre> <p>\u5728\u4f7f\u7528 annotation \u5b9a\u4e49\u5355\u4e2a Pod IP/Mac \u65f6\u9700\u8981\u6ce8\u610f\u4ee5\u4e0b\u51e0\u70b9\uff1a</p> <ol> <li>\u6240\u4f7f\u7528\u7684 IP/Mac \u4e0d\u80fd\u548c\u5df2\u6709\u7684 IP/Mac \u51b2\u7a81\u3002</li> <li>IP \u5fc5\u987b\u5728\u6240\u5c5e\u5b50\u7f51\u7684 CIDR \u5185\u3002</li> <li>\u53ef\u4ee5\u53ea\u6307\u5b9a IP \u6216 Mac\uff0c\u53ea\u6307\u5b9a\u4e00\u4e2a\u65f6\uff0c\u53e6\u4e00\u4e2a\u4f1a\u968f\u673a\u5206\u914d\u3002</li> </ol>"},{"location":"guide/static-ip-mac/#workload-ip-pool","title":"Workload \u901a\u7528 IP Pool \u56fa\u5b9a\u5730\u5740","text":"<p>Kube-OVN \u652f\u6301\u901a\u8fc7 annotation <code>ovn.kubernetes.io/ip_pool</code> \u7ed9 Workload\uff08Deployment/StatefulSet/DaemonSet/Job/CronJob\uff09\u8bbe\u7f6e\u56fa\u5b9a IP\u3002 <code>kube-ovn-controllerr</code> \u4f1a\u81ea\u52a8\u9009\u62e9 <code>ovn.kubernetes.io/ip_pool</code> \u4e2d\u6307\u5b9a\u7684 IP \u5e76\u8fdb\u884c\u51b2\u7a81\u68c0\u6d4b\u3002</p> <p>IP Pool \u7684 Annotation \u9700\u8981\u52a0\u5728 <code>template</code> \u5185\u7684 <code>annotation</code> \u5b57\u6bb5\uff0c\u9664\u4e86 Kubernetes \u5185\u7f6e\u7684 Workload \u7c7b\u578b\uff0c \u5176\u4ed6\u7528\u6237\u81ea\u5b9a\u4e49\u7684 Workload \u4e5f\u53ef\u4ee5\u4f7f\u7528\u540c\u6837\u7684\u65b9\u5f0f\u8fdb\u884c\u56fa\u5b9a\u5730\u5740\u5206\u914d\u3002</p>"},{"location":"guide/static-ip-mac/#deployment-ip","title":"Deployment \u56fa\u5b9a IP \u793a\u4f8b","text":"<pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nname: ippool\nlabels:\napp: ippool\nspec:\nreplicas: 2\nselector:\nmatchLabels:\napp: ippool\ntemplate:\nmetadata:\nlabels:\napp: ippool\nannotations:\novn.kubernetes.io/ip_pool: 10.16.0.15,10.16.0.16,10.16.0.17 // \u53cc\u6808\u5730\u5740\u4f7f\u7528\u5206\u53f7\u8fdb\u884c\u5206\u9694 10.16.0.15,fd00:10:16::000E;10.16.0.16,fd00:10:16::000F;10.16.0.17,fd00:10:16::0010\nspec:\ncontainers:\n- name: ippool\nimage: docker.io/library/nginx:alpine\n</code></pre> <p>\u5bf9 Workload \u4f7f\u7528\u56fa\u5b9a IP \u9700\u8981\u6ce8\u610f\u4ee5\u4e0b\u51e0\u70b9\uff1a</p> <ol> <li><code>ovn.kubernetes.io/ip_pool</code> \u4e2d\u7684 IP \u5e94\u8be5\u5c5e\u4e8e\u6240\u5728\u5b50\u7f51\u7684 CIDR \u5185\u3002</li> <li><code>ovn.kubernetes.io/ip_pool</code> \u4e2d\u7684 IP \u4e0d\u80fd\u548c\u5df2\u4f7f\u7528\u7684 IP \u51b2\u7a81\u3002</li> <li>\u5f53 <code>ovn.kubernetes.io/ip_pool</code> \u4e2d\u7684 IP \u6570\u91cf\u5c0f\u4e8e replicas \u6570\u91cf\u65f6\uff0c\u591a\u51fa\u7684 Pod \u5c06\u65e0\u6cd5\u521b\u5efa\u3002\u4f60\u9700\u8981\u6839\u636e Workload \u7684\u66f4\u65b0\u7b56\u7565\u4ee5\u53ca\u6269\u5bb9\u89c4\u5212\u8c03\u6574 <code>ovn.kubernetes.io/ip_pool</code> \u4e2d IP \u7684\u6570\u91cf\u3002</li> </ol>"},{"location":"guide/static-ip-mac/#statefulset","title":"StatefulSet \u56fa\u5b9a\u5730\u5740","text":"<p>StatefulSet \u9ed8\u8ba4\u652f\u6301\u56fa\u5b9a IP\uff0c\u800c\u4e14\u548c\u5176\u4ed6 Workload \u76f8\u540c\uff0c\u53ef\u4ee5\u4f7f\u7528 <code>ovn.kubernetes.io/ip_pool</code> \u6765\u6307\u5b9a Pod \u4f7f\u7528\u7684 IP \u8303\u56f4\u3002</p> <p>\u7531\u4e8e StatefulSet \u591a\u7528\u4e8e\u6709\u72b6\u6001\u670d\u52a1\uff0c\u5bf9\u7f51\u7edc\u6807\u793a\u7684\u56fa\u5b9a\u6709\u66f4\u9ad8\u7684\u8981\u6c42\uff0cKube-OVN \u505a\u4e86\u7279\u6b8a\u7684\u5f3a\u5316\uff1a</p> <ol> <li>Pod \u4f1a\u6309\u987a\u5e8f\u5206\u914d <code>ovn.kubernetes.io/ip_pool</code> \u4e2d\u7684 IP\u3002\u4f8b\u5982 StatefulSet \u7684\u540d\u5b57\u4e3a web\uff0c\u5219 web-0 \u4f1a\u4f7f\u7528 <code>ovn.kubernetes.io/ip_pool</code> \u4e2d\u7684\u7b2c\u4e00\u4e2a IP\uff0c web-1 \u4f1a\u4f7f\u7528\u7b2c\u4e8c\u4e2a IP\uff0c\u4ee5\u6b64\u7c7b\u63a8\u3002</li> <li>StatefulSet Pod \u5728\u66f4\u65b0\u6216\u5220\u9664\u7684\u8fc7\u7a0b\u4e2d OVN \u4e2d\u7684 logical_switch_port \u4e0d\u4f1a\u5220\u9664\uff0c\u65b0\u751f\u6210\u7684 Pod \u76f4\u63a5\u590d\u7528\u65e7\u7684 interface \u4fe1\u606f\u3002\u56e0\u6b64 Pod \u53ef\u4ee5\u590d\u7528 IP/Mac \u53ca\u5176\u4ed6\u7f51\u7edc\u4fe1\u606f\uff0c\u8fbe\u5230\u548c StatefulSet Volume \u7c7b\u4f3c\u7684\u72b6\u6001\u4fdd\u7559\u529f\u80fd\u3002</li> <li>\u57fa\u4e8e 2 \u7684\u80fd\u529b\uff0c\u5bf9\u4e8e\u6ca1\u6709 <code>ovn.kubernetes.io/ip_pool</code> \u6ce8\u89e3\u7684 StatefulSet\uff0cPod \u7b2c\u4e00\u6b21\u751f\u6210\u65f6\u4f1a\u968f\u673a\u5206\u914d IP/Mac\uff0c\u4e4b\u540e\u5728\u6574\u4e2a StatefulSet \u7684\u751f\u547d\u5468\u671f\u5185\uff0c\u7f51\u7edc\u4fe1\u606f\u90fd\u4f1a\u4fdd\u6301\u56fa\u5b9a\u3002</li> </ol>"},{"location":"guide/static-ip-mac/#statefulset_1","title":"StatefulSet \u793a\u4f8b","text":"<pre><code>apiVersion: apps/v1\nkind: StatefulSet\nmetadata:\nname: web\nspec:\nserviceName: \"nginx\"\nreplicas: 2\nselector:\nmatchLabels:\napp: nginx\ntemplate:\nmetadata:\nlabels:\napp: nginx\nspec:\ncontainers:\n- name: nginx\nimage: docker.io/library/nginx:alpine\nports:\n- containerPort: 80\nname: web\n</code></pre> <p>\u53ef\u4ee5\u5c1d\u8bd5\u5220\u9664 StatefulSet \u4e0b Pod \u89c2\u5bdf Pod IP \u53d8\u5316\u4fe1\u606f\u3002</p>"},{"location":"guide/static-ip-mac/#kubevirt-vm","title":"KubeVirt VM \u56fa\u5b9a\u5730\u5740","text":"<p>\u9488\u5bf9 KubeVirt \u521b\u5efa\u7684 VM \u5b9e\u4f8b\uff0c<code>kube-ovn-controller</code> \u53ef\u4ee5\u6309\u7167\u7c7b\u4f3c StatefulSet Pod \u7684\u65b9\u5f0f\u8fdb\u884c IP \u5730\u5740\u5206\u914d\u548c\u7ba1\u7406\u3002 \u4ee5\u8fbe\u5230 VM \u5b9e\u4f8b\u5728\u751f\u547d\u5468\u671f\u5185\u542f\u505c\uff0c\u5347\u7ea7\uff0c\u8fc1\u79fb\u7b49\u64cd\u4f5c\u8fc7\u7a0b\u4e2d\u5730\u5740\u56fa\u5b9a\u4e0d\u53d8\uff0c\u66f4\u7b26\u865a\u62df\u5316\u5408\u7528\u6237\u7684\u5b9e\u9645\u4f7f\u7528\u4f53\u9a8c\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/subnet/","title":"\u5b50\u7f51\u4f7f\u7528","text":"<p>\u5b50\u7f51\u662f Kube-OVN \u4e2d\u7684\u4e00\u4e2a\u6838\u5fc3\u6982\u5ff5\u548c\u57fa\u672c\u4f7f\u7528\u5355\u5143\uff0cKube-OVN \u4f1a\u4ee5\u5b50\u7f51\u6765\u7ec4\u7ec7 IP \u548c\u7f51\u7edc\u914d\u7f6e\uff0c\u6bcf\u4e2a Namespace \u53ef\u4ee5\u5f52\u5c5e\u4e8e\u7279\u5b9a\u7684\u5b50\u7f51\uff0c Namespace \u4e0b\u7684 Pod \u4f1a\u81ea\u52a8\u4ece\u6240\u5c5e\u7684\u5b50\u7f51\u4e2d\u83b7\u53d6 IP \u5e76\u5171\u4eab\u5b50\u7f51\u7684\u7f51\u7edc\u914d\u7f6e\uff08CIDR\uff0c\u7f51\u5173\u7c7b\u578b\uff0c\u8bbf\u95ee\u63a7\u5236\uff0cNAT \u63a7\u5236\u7b49\uff09\u3002</p> <p>\u548c\u5176\u4ed6 CNI \u7684\u6bcf\u4e2a\u8282\u70b9\u7ed1\u5b9a\u4e00\u4e2a\u5b50\u7f51\u7684\u5b9e\u73b0\u4e0d\u540c\uff0c\u5728 Kube-OVN \u4e2d\u5b50\u7f51\u4e3a\u4e00\u4e2a\u5168\u5c40\u7684\u865a\u62df\u7f51\u7edc\u914d\u7f6e\uff0c\u540c\u4e00\u4e2a\u5b50\u7f51\u7684\u5730\u5740\u53ef\u4ee5\u5206\u5e03\u5728\u4efb\u610f\u4e00\u4e2a\u8282\u70b9\u4e0a\u3002</p> <p></p> <p>Overlay \u548c Underlay \u7684\u5b50\u7f51\u5728\u4f7f\u7528\u548c\u914d\u7f6e\u4e0a\u5b58\u5728\u4e00\u4e9b\u5dee\u5f02\uff0c\u672c\u6587\u6863\u5c06\u4f1a\u4ecb\u7ecd\u4e0d\u540c\u7c7b\u578b\u5b50\u7f51\u7684\u4e00\u4e9b\u5171\u540c\u914d\u7f6e\u548c\u5dee\u5f02\u5316\u529f\u80fd\u3002</p>"},{"location":"guide/subnet/#_2","title":"\u9ed8\u8ba4\u5b50\u7f51","text":"<p>\u4e3a\u4e86\u65b9\u4fbf\u7528\u6237\u7684\u5feb\u901f\u4e0a\u624b\u4f7f\u7528\uff0cKube-OVN \u5185\u7f6e\u4e86\u4e00\u4e2a\u9ed8\u8ba4\u5b50\u7f51\uff0c\u6240\u6709\u672a\u663e\u5f0f\u58f0\u660e\u5b50\u7f51\u5f52\u5c5e\u7684 Namespace \u4f1a\u81ea\u52a8\u4ece\u9ed8\u8ba4\u5b50\u7f51\u4e2d\u5206\u914d IP\uff0c \u5e76\u4f7f\u7528\u9ed8\u8ba4\u5b50\u7f51\u7684\u7f51\u7edc\u4fe1\u606f\u3002\u8be5\u5b50\u7f51\u7684\u914d\u7f6e\u4e3a\u5b89\u88c5\u65f6\u6307\u5b9a\uff0c\u53ef\u4ee5\u53c2\u8003\u5185\u7f6e\u7f51\u7edc\u8bbe\u7f6e\uff0c \u5982\u679c\u8981\u5728\u5b89\u88c5\u540e\u4fee\u6539\u9ed8\u8ba4\u7f51\u7edc\u7684 CIDR \u8bf7\u53c2\u8003\u4fee\u6539\u9ed8\u8ba4\u7f51\u7edc\u3002</p> <p>\u5728 Overlay \u6a21\u5f0f\u4e0b\uff0c\u9ed8\u8ba4\u5b50\u7f51\u4f7f\u7528\u4e86\u5206\u5e03\u5f0f\u7f51\u5173\u5e76\u5bf9\u51fa\u7f51\u6d41\u91cf\u8fdb\u884c NAT \u8f6c\u6362\uff0c\u5176\u884c\u4e3a\u548c Flannel \u7684\u9ed8\u8ba4\u884c\u4e3a\u57fa\u672c\u4e00\u81f4\uff0c \u7528\u6237\u65e0\u9700\u989d\u5916\u7684\u914d\u7f6e\u5373\u53ef\u4f7f\u7528\u5230\u5927\u90e8\u5206\u7684\u7f51\u7edc\u529f\u80fd\u3002</p> <p>\u5728 Underlay \u6a21\u5f0f\u4e0b\uff0c\u9ed8\u8ba4\u5b50\u7f51\u4f7f\u7528\u7269\u7406\u7f51\u5173\u4f5c\u4e3a\u51fa\u7f51\u7f51\u5173\uff0c\u5e76\u5f00\u542f arping \u68c0\u67e5\u7f51\u7edc\u8fde\u901a\u6027\u3002</p>"},{"location":"guide/subnet/#_3","title":"\u67e5\u770b\u9ed8\u8ba4\u5b50\u7f51","text":"<p>\u9ed8\u8ba4\u5b50\u7f51 spec \u4e2d\u7684 default \u5b57\u6bb5\u4e3a true\uff0c\u4e00\u4e2a\u96c6\u7fa4\u4e0b\u53ea\u6709\u4e00\u4e2a\u9ed8\u8ba4\u5b50\u7f51\uff0c\u9ed8\u8ba4\u540d\u4e3a <code>ovn-default</code>\u3002</p> <p>\u67e5\u770b\u9ed8\u8ba4\u5b50\u7f51\uff1a</p> <pre><code># kubectl get subnet ovn-default -o yaml\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n  creationTimestamp: \"2019-08-06T09:33:43Z\"\ngeneration: 1\nname: ovn-default\n  resourceVersion: \"1571334\"\nselfLink: /apis/kubeovn.io/v1/subnets/ovn-default\n  uid: 7e2451f8-fb44-4f7f-b3e0-cfd27f6fd5d6\nspec:\n  cidrBlock: 10.16.0.0/16\n  default: true\nexcludeIps:\n  - 10.16.0.1\n  gateway: 10.16.0.1\n  gatewayType: distributed\n  natOutgoing: true\nprivate: false\nprotocol: IPv4\n</code></pre>"},{"location":"guide/subnet/#join","title":"Join \u5b50\u7f51","text":"<p>\u5728 Kubernetes \u7684\u7f51\u7edc\u89c4\u8303\u4e2d\uff0c\u8981\u6c42 Node \u53ef\u4ee5\u548c\u6240\u6709\u7684 Pod \u76f4\u63a5\u901a\u4fe1\u3002 \u4e3a\u4e86\u5728 Overlay \u7f51\u7edc\u6a21\u5f0f\u4e0b\u8fbe\u5230\u8fd9\u4e2a\u76ee\u7684\uff0c Kube-OVN \u521b\u5efa\u4e86\u4e00\u4e2a <code>join</code> \u5b50\u7f51\uff0c \u5e76\u5728\u6bcf\u4e2a Node \u8282\u70b9\u521b\u5efa\u4e86\u4e00\u5757\u865a\u62df\u7f51\u5361 ovn0 \u63a5\u5165 <code>join</code> \u5b50\u7f51\uff0c\u901a\u8fc7\u8be5\u7f51\u7edc\u5b8c\u6210\u8282\u70b9\u548c Pod \u4e4b\u95f4\u7684\u7f51\u7edc\u4e92\u901a\u3002</p> <p>\u8be5\u5b50\u7f51\u7684\u914d\u7f6e\u4e3a\u5b89\u88c5\u65f6\u6307\u5b9a\uff0c\u53ef\u4ee5\u53c2\u8003\u5185\u7f6e\u7f51\u7edc\u8bbe\u7f6e\uff0c\u5982\u679c\u8981\u5728\u5b89\u88c5\u540e\u4fee\u6539\u3002 join \u5b50\u7f51\u7684 CIDR \u8bf7\u53c2\u8003\u4fee\u6539 Join \u5b50\u7f51</p>"},{"location":"guide/subnet/#join_1","title":"\u67e5\u770b Join \u5b50\u7f51","text":"<p>\u8be5\u5b50\u7f51\u9ed8\u8ba4\u540d\u4e3a <code>join</code> \u4e00\u822c\u65e0\u9700\u5bf9\u8be5\u5b50\u7f51 CIDR \u5916\u7684\u5176\u4ed6\u7f51\u7edc\u914d\u7f6e\u8fdb\u884c\u4fee\u6539\u3002</p> <pre><code># kubectl get subnet join -o yaml\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n  creationTimestamp: \"2019-08-06T09:33:43Z\"\ngeneration: 1\nname: join\n  resourceVersion: \"1571333\"\nselfLink: /apis/kubeovn.io/v1/subnets/join\n  uid: 9c744810-c678-4d50-8a7d-b8ec12ef91b8\nspec:\n  cidrBlock: 100.64.0.0/16\n  default: false\nexcludeIps:\n  - 100.64.0.1\n  gateway: 100.64.0.1\n  gatewayNode: \"\"\ngatewayType: \"\"\nnatOutgoing: false\nprivate: false\nprotocol: IPv4\n</code></pre> <p>\u5728 node \u8282\u70b9\u67e5\u770b ovn0 \u7f51\u5361\uff1a</p> <pre><code># ifconfig ovn0\novn0: flags=4163&lt;UP,BROADCAST,RUNNING,MULTICAST&gt;  mtu 1420\ninet 100.64.0.4  netmask 255.255.0.0  broadcast 100.64.255.255\n        inet6 fe80::800:ff:fe40:5  prefixlen 64  scopeid 0x20&lt;link&gt;\n        ether 0a:00:00:40:00:05  txqueuelen 1000  (Ethernet)\nRX packets 18  bytes 1428 (1.3 KiB)\nRX errors 0  dropped 0  overruns 0  frame 0\nTX packets 19  bytes 1810 (1.7 KiB)\nTX errors 0  dropped 0 overruns 0  carrier 0  collisions 0\n</code></pre>"},{"location":"guide/subnet/#_4","title":"\u521b\u5efa\u81ea\u5b9a\u4e49\u5b50\u7f51","text":"<p>\u8fd9\u91cc\u6211\u4eec\u4ecb\u7ecd\u521b\u5efa\u4e00\u4e2a\u5b50\u7f51\uff0c\u5e76\u5c06\u5176\u548c\u67d0\u4e2a Namespace \u505a\u5173\u8054\u7684\u57fa\u672c\u64cd\u4f5c\uff0c\u66f4\u591a\u9ad8\u7ea7\u914d\u7f6e\u8bf7\u53c2\u8003\u540e\u7eed\u5185\u5bb9\u3002</p>"},{"location":"guide/subnet/#_5","title":"\u521b\u5efa\u5b50\u7f51","text":"<pre><code>cat &lt;&lt;EOF | kubectl create -f -\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n  name: subnet1\nspec:\n  protocol: IPv4\n  cidrBlock: 10.66.0.0/16\n  excludeIps:\n  - 10.66.0.1..10.66.0.10\n  - 10.66.0.101..10.66.0.151\n  gateway: 10.66.0.1\n  gatewayType: distributed\n  natOutgoing: true\n  routeTable: \"\"\n  namespaces:\n  - ns1\n  - ns2\nEOF\n</code></pre> <ul> <li><code>cidrBlock</code>: \u5b50\u7f51 CIDR \u8303\u56f4\uff0c\u540c\u4e00\u4e2a VPC \u4e0b\u7684\u4e0d\u540c Subnet CIDR \u4e0d\u80fd\u91cd\u53e0\u3002</li> <li><code>excludeIps</code>: \u4fdd\u7559\u5730\u5740\u5217\u8868\uff0c\u5bb9\u5668\u7f51\u7edc\u5c06\u4e0d\u4f1a\u81ea\u52a8\u5206\u914d\u5217\u8868\u5185\u7684\u5730\u5740\uff0c\u53ef\u7528\u505a\u56fa\u5b9a IP \u5730\u5740\u5206\u914d\u6bb5\uff0c\u4e5f\u53ef\u5728 Underlay \u6a21\u5f0f\u4e0b\u907f\u514d\u548c\u7269\u7406\u7f51\u7edc\u4e2d\u5df2\u6709\u8bbe\u5907\u51b2\u7a81\u3002</li> <li><code>gateway</code>\uff1a\u8be5\u5b50\u7f51\u7f51\u5173\u5730\u5740\uff0cOverlay \u6a21\u5f0f\u4e0b Kube-OVN \u4f1a\u81ea\u52a8\u5206\u914d\u5bf9\u5e94\u7684\u903b\u8f91\u7f51\u5173\uff0cUnderlay \u6a21\u5f0f\u4e0b\u8be5\u5730\u5740\u9700\u4e3a\u5e95\u5c42\u7269\u7406\u7f51\u5173\u5730\u5740\u3002</li> <li><code>namespaces</code>: \u7ed1\u5b9a\u8be5\u5b50\u7f51\u7684 Namespace \u5217\u8868\uff0c\u7ed1\u5b9a\u540e Namespace \u4e0b\u7684 Pod \u5c06\u4f1a\u4ece\u5f53\u524d\u5b50\u7f51\u5206\u914d\u5730\u5740\u3002</li> <li><code>routeTable</code>: \u5173\u8054\u7684\u8def\u7531\u8868\uff0c\u9ed8\u8ba4\u5173\u8054\u4e3b\u8def\u7531\u8868\uff0c\u8def\u7531\u8868\u5b9a\u4e49\u8bf7\u53c2\u8003\u9759\u6001\u8def\u7531</li> </ul>"},{"location":"guide/subnet/#_6","title":"\u9a8c\u8bc1\u5b50\u7f51\u7ed1\u5b9a\u751f\u6548","text":"<pre><code># kubectl create ns ns1\nnamespace/ns1 created\n\n# kubectl run nginx --image=docker.io/library/nginx:alpine -n ns1\ndeployment.apps/nginx created\n\n# kubectl get pod -n ns1 -o wide\nNAME                     READY   STATUS    RESTARTS   AGE   IP           NODE    NOMINATED NODE   READINESS GATES\nnginx-74d5899f46-n8wtg   1/1     Running   0          10s   10.66.0.11   node1   &lt;none&gt;           &lt;none&gt;\n</code></pre>"},{"location":"guide/subnet/#overlay","title":"Overlay \u5b50\u7f51\u7f51\u5173\u914d\u7f6e","text":"<p>\u8be5\u529f\u80fd\u53ea\u5bf9 Overlay \u6a21\u5f0f\u5b50\u7f51\u751f\u6548\uff0cUnderlay \u7c7b\u578b\u5b50\u7f51\u8bbf\u95ee\u5916\u90e8\u7f51\u7edc\u9700\u8981\u501f\u52a9\u5e95\u5c42\u7269\u7406\u7f51\u5173\u3002</p> <p>Overlay \u5b50\u7f51\u4e0b\u7684 Pod \u9700\u8981\u901a\u8fc7\u7f51\u5173\u6765\u8bbf\u95ee\u96c6\u7fa4\u5916\u90e8\u7f51\u7edc\uff0cKube-OVN \u76ee\u524d\u652f\u6301\u4e24\u79cd\u7c7b\u578b\u7684\u7f51\u5173\uff1a \u5206\u5e03\u5f0f\u7f51\u5173\u548c\u96c6\u4e2d\u5f0f\u7f51\u5173\uff0c\u7528\u6237\u53ef\u4ee5\u5728\u5b50\u7f51\u4e2d\u5bf9\u7f51\u5173\u7684\u7c7b\u578b\u8fdb\u884c\u8c03\u6574\u3002</p> <p>\u4e24\u79cd\u7c7b\u578b\u7f51\u5173\u5747\u652f\u6301 <code>natOutgoing</code> \u8bbe\u7f6e\uff0c\u7528\u6237\u53ef\u4ee5\u9009\u62e9 Pod \u8bbf\u95ee\u5916\u7f51\u65f6\u662f\u5426\u9700\u8981\u8fdb\u884c snat\u3002</p>"},{"location":"guide/subnet/#_7","title":"\u5206\u5e03\u5f0f\u7f51\u5173","text":"<p>\u5b50\u7f51\u7684\u9ed8\u8ba4\u7c7b\u578b\u7f51\u5173\uff0c\u6bcf\u4e2a node \u4f1a\u4f5c\u4e3a\u5f53\u524d node \u4e0a pod \u8bbf\u95ee\u5916\u90e8\u7f51\u7edc\u7684\u7f51\u5173\u3002 \u6570\u636e\u5305\u4f1a\u901a\u8fc7\u672c\u673a\u7684 <code>ovn0</code> \u7f51\u5361\u6d41\u5165\u4e3b\u673a\u7f51\u7edc\u6808\uff0c\u518d\u6839\u636e\u4e3b\u673a\u7684\u8def\u7531\u89c4\u5219\u8fdb\u884c\u51fa\u7f51\u3002 \u5f53 <code>natOutgoing</code> \u4e3a <code>true</code> \u65f6\uff0cPod \u8bbf\u95ee\u5916\u90e8\u7f51\u7edc\u5c06\u4f1a\u4f7f\u7528\u5f53\u524d\u6240\u5728\u5bbf\u4e3b\u673a\u7684 IP\u3002</p> <p></p> <p>\u5b50\u7f51\u793a\u4f8b\uff0c\u5176\u4e2d <code>gatewayType</code> \u5b57\u6bb5\u4e3a <code>distributed</code>\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: distributed\nspec:\nprotocol: IPv4\ncidrBlock: 10.166.0.0/16\ndefault: false\nexcludeIps:\n- 10.166.0.1\ngateway: 10.166.0.1\ngatewayType: distributed\nnatOutgoing: true\n</code></pre>"},{"location":"guide/subnet/#_8","title":"\u96c6\u4e2d\u5f0f\u7f51\u5173","text":"<p>\u5982\u679c\u5e0c\u671b\u5b50\u7f51\u5185\u6d41\u91cf\u8bbf\u95ee\u5916\u7f51\u4f7f\u7528\u56fa\u5b9a\u7684 IP\uff0c\u4ee5\u4fbf\u5ba1\u8ba1\u548c\u767d\u540d\u5355\u7b49\u5b89\u5168\u64cd\u4f5c\uff0c\u53ef\u4ee5\u5728\u5b50\u7f51\u4e2d\u8bbe\u7f6e\u7f51\u5173\u7c7b\u578b\u4e3a\u96c6\u4e2d\u5f0f\u7f51\u5173\u3002 \u5728\u96c6\u4e2d\u5f0f\u7f51\u5173\u6a21\u5f0f\u4e0b\uff0cPod \u8bbf\u95ee\u5916\u7f51\u7684\u6570\u636e\u5305\u4f1a\u9996\u5148\u88ab\u8def\u7531\u5230\u7279\u5b9a\u8282\u70b9\u7684 <code>ovn0</code> \u7f51\u5361\uff0c\u518d\u901a\u8fc7\u4e3b\u673a\u7684\u8def\u7531\u89c4\u5219\u8fdb\u884c\u51fa\u7f51\u3002 \u5f53 <code>natOutgoing</code> \u4e3a <code>true</code> \u65f6\uff0cPod \u8bbf\u95ee\u5916\u90e8\u7f51\u7edc\u5c06\u4f1a\u4f7f\u7528\u7279\u5b9a\u5bbf\u4e3b\u673a\u7684 IP\u3002</p> <p>\u5b50\u7f51\u793a\u4f8b\uff0c\u5176\u4e2d <code>gatewayType</code> \u5b57\u6bb5\u4e3a <code>centralized</code>\uff0c<code>gatewayNode</code> \u4e3a\u7279\u5b9a\u673a\u5668\u5728 Kubernetes \u4e2d\u7684 NodeName\u3002 \u5176\u4e2d <code>gatewayNode</code> \u5b57\u6bb5\u53ef\u4ee5\u4e3a\u9017\u53f7\u5206\u9694\u7684\u591a\u53f0\u4e3b\u673a\u3002</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: centralized\nspec:\nprotocol: IPv4\ncidrBlock: 10.166.0.0/16\ndefault: false\nexcludeIps:\n- 10.166.0.1\ngateway: 10.166.0.1\ngatewayType: centralized\ngatewayNode: \"node1,node2\"\nnatOutgoing: true\n</code></pre> <ul> <li>\u96c6\u4e2d\u5f0f\u7f51\u5173\u5982\u679c\u5e0c\u671b\u6307\u5b9a\u673a\u5668\u7684\u7279\u5b9a\u7f51\u5361\u8fdb\u884c\u51fa\u7f51\uff0c<code>gatewayNode</code> \u53ef\u66f4\u6539\u4e3a <code>kube-ovn-worker:172.18.0.2, kube-ovn-control-plane:172.18.0.3</code> \u683c\u5f0f\u3002</li> <li>\u96c6\u4e2d\u5f0f\u7f51\u5173\u9ed8\u8ba4\u4e3a\u4e3b\u5907\u6a21\u5f0f\uff0c\u53ea\u6709\u4e3b\u8282\u70b9\u8fdb\u884c\u6d41\u91cf\u8f6c\u53d1\uff0c \u5982\u679c\u9700\u8981\u5207\u6362\u4e3a ECMP \u6a21\u5f0f\uff0c\u8bf7\u53c2\u8003\u96c6\u4e2d\u5f0f\u7f51\u5173 ECMP \u5f00\u542f\u8bbe\u7f6e\u3002</li> <li>\u4ece Kube-OVN v1.12.0 \u7248\u672c\u5f00\u59cb\uff0c\u5728 subnet crd \u5b9a\u4e49\u4e2d\u589e\u52a0\u4e86 spec \u5b57\u6bb5 <code>enableEcmp</code>\uff0c\u5c06\u96c6\u4e2d\u5f0f\u5b50\u7f51 ECMP \u5f00\u5173\u63a7\u5236\u8fc1\u79fb\u5230\u5b50\u7f51\u5c42\u7ea7\uff0c\u53ef\u4ee5\u57fa\u4e8e\u4e0d\u540c\u7684\u5b50\u7f51\u5206\u522b\u8bbe\u7f6e\u662f\u5426\u5f00\u542f ECMP \u6a21\u5f0f\u3002\u539f\u6709\u7684 <code>kube-ovn-controller</code> Deployment \u4e2d\u7684 <code>enable-ecmp</code> \u53c2\u6570\u4e0d\u518d\u4f7f\u7528\u3002\u4e4b\u524d\u7248\u672c\u5347\u7ea7\u5230 v1.12.0 \u4e4b\u540e\uff0c\u5b50\u7f51\u5f00\u5173\u4f1a\u81ea\u52a8\u7ee7\u627f\u539f\u6709\u7684\u5168\u5c40\u5f00\u5173\u53c2\u6570\u53d6\u503c\u3002</li> </ul>"},{"location":"guide/subnet/#acl","title":"\u5b50\u7f51 ACL \u8bbe\u7f6e","text":"<p>\u5bf9\u4e8e\u6709\u7ec6\u7c92\u5ea6 ACL \u63a7\u5236\u7684\u573a\u666f\uff0cKube-OVN \u7684 Subnet \u63d0\u4f9b\u4e86 ACL \u89c4\u5219\u7684\u8bbe\u7f6e\uff0c\u53ef\u4ee5\u5b9e\u73b0\u7f51\u7edc\u89c4\u5219\u7684\u7cbe\u7ec6\u63a7\u5236\u3002</p> <p>Subnet \u4e2d\u7684 ACL \u89c4\u5219\u548c OVN \u7684 ACL \u89c4\u5219\u4e00\u81f4\uff0c\u76f8\u5173\u5b57\u6bb5\u5185\u5bb9\u53ef\u4ee5\u53c2\u8003 ovn-nb ACL Table\uff0c <code>match</code> \u5b57\u6bb5\u652f\u6301\u7684\u5b57\u6bb5\u53ef\u53c2\u8003 ovn-sb Logical Flow Table\u3002</p> <p>\u5141\u8bb8 IP \u5730\u5740\u4e3a <code>10.10.0.2</code> \u7684 Pod \u8bbf\u95ee\u6240\u6709\u5730\u5740\uff0c\u4f46\u4e0d\u5141\u8bb8\u5176\u4ed6\u5730\u5740\u4e3b\u52a8\u8bbf\u95ee\u81ea\u5df1\u7684 ACL \u89c4\u5219\u793a\u4f8b\u5982\u4e0b\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: acl\nspec:\nacls:\n- action: drop\ndirection: to-lport\nmatch: ip4.dst == 10.10.0.2 &amp;&amp; ip\npriority: 1002\n- action: allow-related\ndirection: from-lport\nmatch: ip4.src == 10.10.0.2 &amp;&amp; ip\npriority: 1002\ncidrBlock: 10.10.0.0/24\n</code></pre>"},{"location":"guide/subnet/#_9","title":"\u5b50\u7f51\u9694\u79bb\u8bbe\u7f6e","text":"<p>\u5b50\u7f51 ACL \u7684\u529f\u80fd\u53ef\u4ee5\u8986\u76d6\u5b50\u7f51\u9694\u79bb\u7684\u529f\u80fd\uff0c\u5e76\u6709\u66f4\u597d\u7684\u7075\u6d3b\u6027\uff0c\u6211\u4eec\u63a8\u8350\u4f7f\u7528\u5b50\u7f51 ACL \u6765\u505a\u76f8\u5e94\u7684\u914d\u7f6e\u3002</p> <p>\u9ed8\u8ba4\u60c5\u51b5\u4e0b Kube-OVN \u521b\u5efa\u7684\u5b50\u7f51\u4e4b\u95f4\u53ef\u4ee5\u76f8\u4e92\u901a\u4fe1\uff0cPod \u4e5f\u53ef\u4ee5\u901a\u8fc7\u7f51\u5173\u8bbf\u95ee\u5916\u90e8\u7f51\u7edc\u3002</p> <p>\u5982\u9700\u5bf9\u5b50\u7f51\u95f4\u7684\u8bbf\u95ee\u8fdb\u884c\u63a7\u5236\uff0c\u53ef\u4ee5\u5728\u5b50\u7f51 CRD \u4e2d\u5c06 <code>private</code> \u8bbe\u7f6e\u4e3a true\uff0c\u5219\u8be5\u5b50\u7f51\u5c06\u548c\u5176\u4ed6\u5b50\u7f51\u4ee5\u53ca\u5916\u90e8\u7f51\u7edc\u9694\u79bb\uff0c \u53ea\u80fd\u8fdb\u884c\u5b50\u7f51\u5185\u90e8\u7684\u901a\u4fe1\u3002\u5982\u9700\u5f00\u767d\u540d\u5355\uff0c\u53ef\u4ee5\u901a\u8fc7 <code>allowSubnets</code> \u8fdb\u884c\u8bbe\u7f6e\u3002<code>allowSubnets</code> \u5185\u7684\u7f51\u6bb5\u548c\u8be5\u5b50\u7f51\u53ef\u4ee5\u53cc\u5411\u4e92\u8bbf\u3002</p>"},{"location":"guide/subnet/#_10","title":"\u5f00\u542f\u8bbf\u95ee\u63a7\u5236\u7684\u5b50\u7f51\u793a\u4f8b","text":"<pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: private\nspec:\nprotocol: IPv4\ndefault: false\nnamespaces:\n- ns1\n- ns2\ncidrBlock: 10.69.0.0/16\nprivate: true\nallowSubnets:\n- 10.16.0.0/16\n- 10.18.0.0/16\n</code></pre>"},{"location":"guide/subnet/#underlay","title":"Underlay \u76f8\u5173\u9009\u9879","text":"<p>\u8be5\u90e8\u5206\u529f\u80fd\u53ea\u5bf9 Underlay \u7c7b\u578b\u5b50\u7f51\u751f\u6548\u3002</p> <ul> <li><code>vlan</code>: \u5982\u679c\u4f7f\u7528 Underlay \u7f51\u7edc\uff0c\u8be5\u5b57\u6bb5\u7528\u6765\u63a7\u5236\u8be5 Subnet \u548c\u54ea\u4e2a Vlan CR \u8fdb\u884c\u7ed1\u5b9a\u3002\u8be5\u9009\u9879\u9ed8\u8ba4\u4e3a\u7a7a\u5b57\u7b26\u4e32\uff0c\u5373\u4e0d\u4f7f\u7528 Underlay \u7f51\u7edc\u3002</li> <li><code>logicalGateway</code>: \u4e00\u4e9b Underlay \u73af\u5883\u4e3a\u7eaf\u4e8c\u5c42\u7f51\u7edc\uff0c\u4e0d\u5b58\u5728\u7269\u7406\u7684\u4e09\u5c42\u7f51\u5173\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\u53ef\u4ee5\u501f\u52a9 OVN \u672c\u8eab\u7684\u80fd\u529b\u8bbe\u7f6e\u4e00\u4e2a\u865a\u62df\u7f51\u5173\uff0c\u5c06 Underlay   \u548c Overlay \u7f51\u7edc\u6253\u901a\u3002\u9ed8\u8ba4\u503c\u4e3a\uff1a<code>false</code>\u3002</li> </ul>"},{"location":"guide/subnet/#_11","title":"\u7f51\u5173\u68c0\u67e5\u8bbe\u7f6e","text":"<p>\u9ed8\u8ba4\u60c5\u51b5\u4e0b <code>kube-ovn-cni</code> \u5728\u542f\u52a8 Pod \u540e\u4f1a\u4f7f\u7528 ICMP \u6216 ARP \u534f\u8bae\u8bf7\u6c42\u7f51\u5173\u5e76\u7b49\u5f85\u8fd4\u56de\uff0c \u4ee5\u9a8c\u8bc1\u7f51\u7edc\u5de5\u4f5c\u6b63\u5e38\uff0c\u5728\u90e8\u5206 Underlay \u73af\u5883\u7f51\u5173\u65e0\u6cd5\u54cd\u5e94 ARP \u8bf7\u6c42\uff0c\u6216\u65e0\u9700\u7f51\u7edc\u5916\u90e8\u8054\u901a\u7684\u573a\u666f \u53ef\u4ee5\u5173\u95ed\u7f51\u5173\u68c0\u67e5\u3002</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: disable-gw-check\nspec:\ndisableGatewayCheck: true\n</code></pre>"},{"location":"guide/subnet/#multicast-snoop","title":"Multicast-Snoop \u914d\u7f6e","text":"<p>\u9ed8\u8ba4\u60c5\u51b5\u4e0b subnet \u4e0b\u7684 Pod \u5982\u679c\u53d1\u9001\u7ec4\u64ad\u62a5\u6587\uff0cOVN \u7684\u9ed8\u8ba4\u884c\u4e3a\u662f\u4f1a\u5e7f\u64ad\u7ec4\u64ad\u62a5\u6587\u5230\u5b50\u7f51\u4e0b\u6240\u6709\u7684 Pod\u3002\u5982\u679c\u5f00\u542fsubnet \u7684 multicast snoop \u5f00\u5173\uff0cOVN \u4f1a\u6839\u636e <code>South Database</code> \u4e2d\u7684\u7ec4\u64ad\u8868 <code>Multicast_Group</code> \u6765\u8fdb\u884c\u8f6c\u53d1\uff0c\u800c\u4e0d\u5728\u8fdb\u884c\u5e7f\u64ad\u3002</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: sample1\nspec:\nenableMulticastSnoop: true\n</code></pre>"},{"location":"guide/subnet/#_12","title":"\u5176\u4ed6\u9ad8\u7ea7\u8bbe\u7f6e","text":"<ul> <li>IP \u6c60\u4f7f\u7528</li> <li>\u9ed8\u8ba4 VPC NAT \u7b56\u7565\u89c4\u5219</li> <li>QoS \u8bbe\u7f6e</li> <li>\u591a\u7f51\u5361\u7ba1\u7406</li> <li>DHCP \u9009\u9879</li> <li>\u5916\u90e8\u7f51\u5173\u8bbe\u7f6e</li> <li>\u96c6\u7fa4\u4e92\u8054\u8bbe\u7f6e</li> <li>\u865a\u62df IP \u8bbe\u7f6e</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/vpc-qos/","title":"VPC QoS","text":"<p>Kube-OVN \u652f\u6301\u4f7f\u7528 QoSPolicy CRD \u5bf9\u81ea\u5b9a\u4e49 VPC \u7684\u6d41\u91cf\u901f\u7387\u8fdb\u884c\u9650\u5236\u3002</p>"},{"location":"guide/vpc-qos/#eip-qos","title":"EIP QoS","text":"<p>\u5bf9 EIP \u8fdb\u884c\u9650\u901f\uff0c\u9650\u901f\u503c\u4e3a 1Mbps\uff0c\u4f18\u5148\u7ea7\u4e3a 1\uff0c\u8fd9\u91cc <code>shared=false</code>\uff0c\u8868\u793a\u8fd9\u4e2a QoSPolicy \u53ea\u80fd\u7ed9\u8fd9\u4e2a EIP \u4f7f\u7528\u4e14\u652f\u6301\u52a8\u6001\u4fee\u6539 QoSPolicy \u53bb\u53d8\u66f4 QoS \u89c4\u5219\u3002</p> <p>QoSPolicy \u914d\u7f6e\u5982\u4e0b\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: QoSPolicy\nmetadata:\nname: qos-eip-example\nspec:\nshared: false\nbindingType: EIP\nbandwidthLimitRules:\n- name: eip-ingress\nrateMax: \"1\" # Mbps\nburstMax: \"1\" # Mbps\npriority: 1\ndirection: ingress\n- name: eip-egress\nrateMax: \"1\" # Mbps\nburstMax: \"1\" # Mbps\npriority: 1\ndirection: egress\n</code></pre> <p>IptablesEIP \u914d\u7f6e\u5982\u4e0b\uff1a</p> <pre><code>kind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-1\nspec:\nnatGwDp: gw1\nqosPolicy: qos-eip-example\n</code></pre> <p><code>.spec.qosPolicy</code>\u7684\u503c\u652f\u6301\u521b\u5efa\u65f6\u4f20\u5165\uff0c\u4e5f\u652f\u6301\u521b\u5efa\u540e\u4fee\u6539\u3002</p>"},{"location":"guide/vpc-qos/#qos-eip","title":"\u67e5\u770b\u5df2\u542f\u7528 QoS \u7684 EIP","text":"<p>\u901a\u8fc7<code>label</code>\u67e5\u770b\u5df2\u7ecf\u8bbe\u7f6e\u5bf9\u5e94 qos \u7684 eip\uff1a</p> <pre><code># kubectl get eip  -l ovn.kubernetes.io/qos=qos-eip-example\nNAME    IP             MAC                 NAT   NATGWDP   READY\neip-1   172.18.11.24   00:00:00:34:41:0B   fip   gw1       true\n</code></pre>"},{"location":"guide/vpc-qos/#vpc-natgw-net1-qos","title":"VPC NATGW net1 \u7f51\u5361 QoS","text":"<p>\u5bf9 VPC NATGW \u7684 net1 \u7f51\u5361\u901f\u7387\u8fdb\u884c\u9650\u5236\uff0c\u9650\u901f\u503c\u4e3a 10Mbps\uff0c\u4f18\u5148\u7ea7\u4e3a 3\uff0c\u8fd9\u91cc <code>shared=true</code>\uff0c\u8868\u793a\u8fd9\u4e2a QoSPolicy \u53ef\u4ee5\u540c\u65f6\u7ed9\u591a\u4e2a\u8d44\u6e90\u4f7f\u7528\uff0c\u8fd9\u79cd\u573a\u666f\u4e0b\u4e0d\u5141\u8bb8\u4fee\u6539 QoSPolicy \u7684\u5185\u5bb9\u3002</p> <p>QoSPolicy \u914d\u7f6e\u5982\u4e0b\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: QoSPolicy\nmetadata:\nname: qos-natgw-example\nspec:\nshared: true\nbindingType: NATGW\nbandwidthLimitRules:\n- name: net1-ingress\ninterface: net1\nrateMax: \"10\" # Mbps\nburstMax: \"10\" # Mbps\npriority: 3\ndirection: ingress\n- name: net1-egress\ninterface: net1\nrateMax: \"10\" # Mbps\nburstMax: \"10\" # Mbps\npriority: 3\ndirection: egress\n</code></pre> <p>VpcNatGateway \u914d\u7f6e\u5982\u4e0b\uff1a</p> <pre><code>kind: VpcNatGateway\napiVersion: kubeovn.io/v1\nmetadata:\nname: gw1\nspec:\nvpc: test-vpc-1\nsubnet: net1\nlanIp: 10.0.1.254\nqosPolicy: qos-natgw-example\nselector:\n- \"kubernetes.io/hostname: kube-ovn-worker\"\n- \"kubernetes.io/os: linux\"\n</code></pre> <p><code>.spec.qosPolicy</code>\u7684\u503c\u652f\u6301\u521b\u5efa\u4f20\u5165\uff0c\u4e5f\u652f\u6301\u540e\u7eed\u4fee\u6539\u3002</p>"},{"location":"guide/vpc-qos/#net1-qos","title":"net1 \u7f51\u5361\u7279\u5b9a\u6d41\u91cf QoS","text":"<p>\u5bf9 net1 \u7f51\u5361\u4e0a\u7279\u5b9a\u6d41\u91cf\u8fdb\u884c\u9650\u901f\uff0c\u9650\u901f\u503c\u4e3a 5Mbps\uff0c\u4f18\u5148\u7ea7\u4e3a 2\uff0c\u8fd9\u91cc <code>shared=true</code>\uff0c\u8868\u793a\u8fd9\u4e2a QoSPolicy  \u53ef\u4ee5\u540c\u65f6\u7ed9\u591a\u4e2a\u8d44\u6e90\u4f7f\u7528\uff0c\u6b64\u65f6\u4e0d\u5141\u8bb8\u4fee\u6539 QoSPolicy \u7684\u5185\u5bb9\u3002</p> <p>QoSPolicy \u914d\u7f6e\u5982\u4e0b\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: QoSPolicy\nmetadata:\nname: qos-natgw-example\nspec:\nshared: true\nbindingType: NATGW\nbandwidthLimitRules:\n- name: net1-extip-ingress\ninterface: net1\nrateMax: \"5\" # Mbps\nburstMax: \"5\" # Mbps\npriority: 2\ndirection: ingress\nmatchType: ip\nmatchValue: src 172.18.11.22/32\n- name: net1-extip-egress\ninterface: net1\nrateMax: \"5\" # Mbps\nburstMax: \"5\" # Mbps\npriority: 2\ndirection: egress\nmatchType: ip\nmatchValue: dst 172.18.11.23/32\n</code></pre> <p>VpcNatGateway \u914d\u7f6e\u5982\u4e0b\uff1a</p> <pre><code>kind: VpcNatGateway\napiVersion: kubeovn.io/v1\nmetadata:\nname: gw1\nspec:\nvpc: test-vpc-1\nsubnet: net1\nlanIp: 10.0.1.254\nqosPolicy: qos-natgw-example\nselector:\n- \"kubernetes.io/hostname: kube-ovn-worker\"\n- \"kubernetes.io/os: linux\"\n</code></pre>"},{"location":"guide/vpc-qos/#qos-natgw","title":"\u67e5\u770b\u5df2\u542f\u7528 QoS \u7684 NATGW","text":"<p>\u901a\u8fc7<code>label</code>\u67e5\u770b\u5df2\u7ecf\u8bbe\u7f6e\u5bf9\u5e94 qos \u7684 eip\uff1a</p> <pre><code># kubectl get vpc-nat-gw  -l ovn.kubernetes.io/qos=qos-natgw-example\nNAME   VPC          SUBNET   LANIP\ngw1    test-vpc-1   net1     10.0.1.254\n</code></pre>"},{"location":"guide/vpc-qos/#qos","title":"\u67e5\u770b qos \u89c4\u5219","text":"<pre><code># kubectl get qos -A\nNAME                SHARED   BINDINGTYPE\nqos-eip-example     false    EIP\nqos-natgw-example   true     NATGW\n</code></pre>"},{"location":"guide/vpc-qos/#_1","title":"\u9650\u5236","text":"<ul> <li>\u53ea\u6709\u5728\u672a\u4f7f\u7528\u65f6\u624d\u80fd\u5220\u9664 QoS \u7b56\u7565\u3002\u56e0\u6b64\uff0c\u5728\u5220\u9664 QoS \u7b56\u7565\u4e4b\u524d\uff0c\u8bf7\u5148\u67e5\u770b\u5df2\u542f\u7528 QoS \u7684 EIP \u548c NATGW\uff0c\u53bb\u6389\u5b83\u4eec\u7684 <code>spec.qosPolicy</code> \u914d\u7f6e\u3002</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/vpc/","title":"VPC \u4f7f\u7528","text":"<p>Kube-OVN \u652f\u6301\u591a\u79df\u6237\u9694\u79bb\u7ea7\u522b\u7684 VPC \u7f51\u7edc\u3002\u4e0d\u540c VPC \u7f51\u7edc\u76f8\u4e92\u72ec\u7acb\uff0c\u53ef\u4ee5\u5206\u522b\u914d\u7f6e Subnet \u7f51\u6bb5\uff0c \u8def\u7531\u7b56\u7565\uff0c\u5b89\u5168\u7b56\u7565\uff0c\u51fa\u7f51\u7f51\u5173\uff0cEIP \u7b49\u914d\u7f6e\u3002</p> <p>VPC \u4e3b\u8981\u7528\u4e8e\u6709\u591a\u79df\u6237\u7f51\u7edc\u5f3a\u9694\u79bb\u7684\u573a\u666f\uff0c\u90e8\u5206 Kubernetes \u7f51\u7edc\u529f\u80fd\u5728\u591a\u79df\u6237\u7f51\u7edc\u4e0b\u5b58\u5728\u51b2\u7a81\u3002 \u4f8b\u5982\u8282\u70b9\u548c Pod \u4e92\u8bbf\uff0cNodePort \u529f\u80fd\uff0c\u57fa\u4e8e\u7f51\u7edc\u8bbf\u95ee\u7684\u5065\u5eb7\u68c0\u67e5\u548c DNS \u80fd\u529b\u5728\u591a\u79df\u6237\u7f51\u7edc\u573a\u666f\u6682\u4e0d\u652f\u6301\u3002 \u4e3a\u4e86\u65b9\u4fbf\u5e38\u89c1 Kubernetes \u7684\u4f7f\u7528\u573a\u666f\uff0cKube-OVN \u9ed8\u8ba4 VPC \u505a\u4e86\u7279\u6b8a\u8bbe\u8ba1\uff0c\u8be5 VPC \u4e0b\u7684 Subnet \u53ef\u4ee5\u6ee1\u8db3 Kubernetes \u89c4\u8303\u3002\u7528\u6237\u81ea\u5b9a\u4e49 VPC \u652f\u6301\u672c\u6587\u6863\u4ecb\u7ecd\u7684\u9759\u6001\u8def\u7531\uff0cEIP \u548c NAT \u7f51\u5173\u7b49\u529f\u80fd\u3002 \u5e38\u89c1\u9694\u79bb\u9700\u6c42\u53ef\u901a\u8fc7\u9ed8\u8ba4 VPC \u4e0b\u7684\u7f51\u7edc\u7b56\u7565\u548c\u5b50\u7f51 ACL \u5b9e\u73b0\uff0c\u5728\u4f7f\u7528\u81ea\u5b9a\u4e49 VPC \u524d\u8bf7\u660e\u786e\u662f\u5426\u9700\u8981 VPC \u7ea7\u522b\u7684\u9694\u79bb\uff0c\u5e76\u4e86\u89e3\u81ea\u5b9a\u4e49 VPC \u4e0b\u7684\u9650\u5236\u3002 \u5728 Underlay \u7f51\u7edc\u4e0b\uff0c\u7269\u7406\u4ea4\u6362\u673a\u8d1f\u8d23\u6570\u636e\u9762\u8f6c\u53d1\uff0cVPC \u65e0\u6cd5\u5bf9 Underlay \u5b50\u7f51\u8fdb\u884c\u9694\u79bb\u3002</p> <p></p>"},{"location":"guide/vpc/#vpc_1","title":"\u521b\u5efa\u81ea\u5b9a\u4e49 VPC","text":"<p>\u521b\u5efa\u4e24\u4e2a VPC\uff1a</p> <pre><code>kind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: test-vpc-1\nspec:\nnamespaces:\n- ns1\n---\nkind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: test-vpc-2\nspec:\nnamespaces:\n- ns2\n</code></pre> <ul> <li><code>namespaces</code> \u53ef\u4ee5\u9650\u5b9a\u53ea\u6709\u54ea\u4e9b Namespace \u53ef\u4ee5\u4f7f\u7528\u5f53\u524d VPC\uff0c\u82e5\u4e3a\u7a7a\u5219\u4e0d\u9650\u5b9a\u3002</li> </ul> <p>\u521b\u5efa\u4e24\u4e2a\u5b50\u7f51\uff0c\u5206\u5c5e\u4e24\u4e2a\u4e0d\u540c\u7684 VPC \u5e76\u6709\u76f8\u540c\u7684 CIDR:</p> <pre><code>kind: Subnet\napiVersion: kubeovn.io/v1\nmetadata:\nname: net1\nspec:\nvpc: test-vpc-1\ncidrBlock: 10.0.1.0/24\nprotocol: IPv4\nnamespaces:\n- ns1\n---\nkind: Subnet\napiVersion: kubeovn.io/v1\nmetadata:\nname: net2\nspec:\nvpc: test-vpc-2\ncidrBlock: 10.0.1.0/24\nprotocol: IPv4\nnamespaces:\n- ns2\n</code></pre> <p>\u5206\u522b\u5728\u4e24\u4e2a Namespace \u4e0b\u521b\u5efa Pod:</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nannotations:\novn.kubernetes.io/logical_switch: net1\nnamespace: ns1\nname: vpc1-pod\nspec:\ncontainers:\n- name: vpc1-pod\nimage: docker.io/library/nginx:alpine\n---\napiVersion: v1\nkind: Pod\nmetadata:\nannotations:\novn.kubernetes.io/logical_switch: net2\nnamespace: ns2\nname: vpc2-pod\nspec:\ncontainers:\n- name: vpc2-pod\nimage: docker.io/library/nginx:alpine\n</code></pre> <p>\u8fd0\u884c\u6210\u529f\u540e\u53ef\u89c2\u5bdf\u4e24\u4e2a Pod \u5730\u5740\u5c5e\u4e8e\u540c\u4e00\u4e2a CIDR\uff0c\u4f46\u7531\u4e8e\u8fd0\u884c\u5728\u4e0d\u540c\u7684\u79df\u6237 VPC\uff0c\u4e24\u4e2a Pod \u65e0\u6cd5\u76f8\u4e92\u8bbf\u95ee\u3002</p>"},{"location":"guide/vpc/#vpc-pod-livenessprobe-readinessprobe","title":"\u81ea\u5b9a\u4e49 VPC Pod \u652f\u6301 livenessProbe \u548c readinessProbe","text":"<p>\u7531\u4e8e\u5e38\u89c4\u914d\u7f6e\u4e0b\u81ea\u5b9a\u4e49 VPC \u4e0b\u7684 Pod \u548c\u8282\u70b9\u7684\u7f51\u7edc\u4e4b\u95f4\u5e76\u4e0d\u4e92\u901a\uff0c\u6240\u4ee5 kubelet \u53d1\u9001\u7684\u63a2\u6d4b\u62a5\u6587\u65e0\u6cd5\u5230\u8fbe\u81ea\u5b9a VPC \u5185\u7684 Pod\u3002Kube-OVN \u901a\u8fc7 TProxy \u5c06 kubelet \u53d1\u9001\u7684\u63a2\u6d4b\u62a5\u6587\u91cd\u5b9a\u5411\u5230\u81ea\u5b9a\u4e49 VPC \u5185\u7684 Pod\uff0c\u4ece\u800c\u5b9e\u73b0\u8fd9\u4e00\u529f\u80fd\u3002</p> <p>\u914d\u7f6e\u65b9\u6cd5\u5982\u4e0b\uff0c\u5728 Daemonset <code>kube-ovn-cni</code> \u4e2d\u589e\u52a0\u53c2\u6570 <code>--enable-tproxy=true</code>\uff1a</p> <pre><code>spec:\ntemplate:\nspec:\ncontainers:\n- args:\n- --enable-tproxy=true\n</code></pre> <p>\u8be5\u529f\u80fd\u9650\u5236\u6761\u4ef6\uff1a</p> <ol> <li>\u5f53\u540c\u4e00\u4e2a\u8282\u70b9\u4e0b\u51fa\u73b0\u4e0d\u540c VPC \u4e0b\u7684 Pod \u5177\u6709\u76f8\u540c\u7684 IP\uff0c\u63a2\u6d4b\u529f\u80fd\u5931\u6548\u3002</li> <li>\u76ee\u524d\u6682\u65f6\u53ea\u652f\u6301 <code>tcpSocket</code> \u548c <code>httpGet</code> \u4e24\u79cd\u63a2\u6d4b\u65b9\u5f0f\u3002</li> </ol>"},{"location":"guide/vpc/#vpc_2","title":"\u521b\u5efa VPC \u7f51\u5173","text":"<p>\u81ea\u5b9a\u4e49 VPC \u4e0b\u7684\u5b50\u7f51\u4e0d\u652f\u6301\u9ed8\u8ba4 VPC \u4e0b\u7684\u5206\u5e03\u5f0f\u7f51\u5173\u548c\u96c6\u4e2d\u5f0f\u7f51\u5173\u3002</p> <p>VPC \u5185\u5bb9\u5668\u8bbf\u95ee\u5916\u90e8\u7f51\u7edc\u9700\u8981\u901a\u8fc7 VPC \u7f51\u5173\uff0cVPC \u7f51\u5173\u53ef\u4ee5\u6253\u901a\u7269\u7406\u7f51\u7edc\u548c\u79df\u6237\u7f51\u7edc\uff0c\u5e76\u63d0\u4f9b \u6d6e\u52a8 IP\uff0cSNAT \u548c DNAT \u529f\u80fd\u3002</p> <p>VPC \u7f51\u5173\u529f\u80fd\u4f9d\u8d56 Multus-CNI \u7684\u591a\u7f51\u5361\u529f\u80fd\uff0c\u5b89\u88c5\u8bf7\u53c2\u8003 multus-cni\u3002</p>"},{"location":"guide/vpc/#_1","title":"\u914d\u7f6e\u5916\u90e8\u7f51\u7edc","text":"<pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: ovn-vpc-external-network\nspec:\nprotocol: IPv4\nprovider: ovn-vpc-external-network.kube-system\ncidrBlock: 192.168.0.0/24\ngateway: 192.168.0.1  # IP address of the physical gateway\nexcludeIps:\n- 192.168.0.1..192.168.0.10\n---\napiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: ovn-vpc-external-network\nnamespace: kube-system\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.0\",\n\"type\": \"macvlan\",\n\"master\": \"eth1\",\n\"mode\": \"bridge\",\n\"ipam\": {\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"ovn-vpc-external-network.kube-system\"\n}\n}'\n</code></pre> <ul> <li>\u8be5 Subnet \u7528\u6765\u7ba1\u7406\u53ef\u7528\u7684\u5916\u90e8\u5730\u5740\uff0c\u7f51\u6bb5\u5185\u7684\u5730\u5740\u5c06\u4f1a\u901a\u8fc7 Macvlan \u5206\u914d\u7ed9 VPC \u7f51\u5173\uff0c\u8bf7\u548c\u7f51\u7edc\u7ba1\u7406\u6c9f\u901a\u7ed9\u51fa\u53ef\u7528\u7684\u7269\u7406\u6bb5 IP\u3002</li> <li>VPC \u7f51\u5173\u4f7f\u7528 Macvlan \u505a\u7269\u7406\u7f51\u7edc\u914d\u7f6e\uff0c<code>NetworkAttachmentDefinition</code> \u7684 <code>master</code> \u9700\u4e3a\u5bf9\u5e94\u7269\u7406\u7f51\u8def\u7f51\u5361\u7684\u7f51\u5361\u540d\u3002</li> <li><code>name</code> \u5916\u90e8\u7f51\u7edc\u540d\u79f0\u3002</li> </ul> <p>\u5728 Macvlan \u6a21\u5f0f\u4e0b\uff0c\u9644\u5c5e\u7f51\u5361\u4f1a\u5c06\u6570\u636e\u5305\u76f4\u63a5\u901a\u8fc7\u8be5\u8282\u70b9\u7f51\u5361\u5bf9\u5916\u53d1\u9001\uff0cL2/L3 \u5c42\u9762\u7684\u8f6c\u53d1\u80fd\u529b\u9700\u8981\u4f9d\u8d56\u5e95\u5c42\u7f51\u7edc\u8bbe\u5907\u3002 \u9700\u8981\u9884\u5148\u5728\u5e95\u5c42\u7f51\u7edc\u8bbe\u5907\u914d\u7f6e\u5bf9\u5e94\u7684\u7f51\u5173\u3001Vlan \u548c\u5b89\u5168\u7b56\u7565\u7b49\u914d\u7f6e\u3002</p> <ol> <li>\u5bf9\u4e8e OpenStack \u7684 VM \u73af\u5883\uff0c\u9700\u8981\u5c06\u5bf9\u5e94\u7f51\u7edc\u7aef\u53e3\u7684 <code>PortSecurity</code> \u5173\u95ed\u3002</li> <li>\u5bf9\u4e8e VMware \u7684 vSwitch \u7f51\u7edc\uff0c\u9700\u8981\u5c06 <code>MAC Address Changes</code>, <code>Forged Transmits</code> \u548c <code>Promiscuous Mode Operation</code> \u8bbe\u7f6e\u4e3a <code>allow</code>\u3002</li> <li>\u5bf9\u4e8e Hyper-V \u865a\u62df\u5316\uff0c\u9700\u8981\u5f00\u542f\u865a\u62df\u673a\u7f51\u5361\u9ad8\u7ea7\u529f\u80fd\u4e2d\u7684 <code>MAC Address Spoofing</code>\u3002</li> <li>\u516c\u6709\u4e91\uff0c\u4f8b\u5982 AWS\u3001GCE\u3001\u963f\u91cc\u4e91\u7b49\u7531\u4e8e\u4e0d\u652f\u6301\u7528\u6237\u81ea\u5b9a\u4e49 Mac \u65e0\u6cd5\u652f\u6301 Macvlan \u6a21\u5f0f\u7f51\u7edc\u3002</li> <li>\u7531\u4e8e Macvlan \u672c\u8eab\u7684\u9650\u5236\uff0cMacvlan \u5b50\u63a5\u53e3\u65e0\u6cd5\u8bbf\u95ee\u7236\u63a5\u53e3\u5730\u5740\u3002</li> <li>\u5982\u679c\u7269\u7406\u7f51\u5361\u5bf9\u5e94\u4ea4\u6362\u673a\u63a5\u53e3\u4e3a Trunk \u6a21\u5f0f\uff0c\u9700\u8981\u5728\u8be5\u7f51\u5361\u4e0a\u521b\u5efa\u5b50\u63a5\u53e3\u518d\u63d0\u4f9b\u7ed9 Macvlan \u4f7f\u7528\u3002</li> </ol>"},{"location":"guide/vpc/#vpc_3","title":"\u5f00\u542f VPC \u7f51\u5173\u529f\u80fd","text":"<p>VPC \u7f51\u5173\u529f\u80fd\u9700\u8981\u901a\u8fc7 <code>kube-system</code> \u4e0b\u7684 <code>ovn-vpc-nat-gw-config</code> \u5f00\u542f\uff1a</p> <pre><code>---\nkind: ConfigMap\napiVersion: v1\nmetadata:\nname: ovn-vpc-nat-config\nnamespace: kube-system\ndata:\nimage: 'docker.io/kubeovn/vpc-nat-gateway:v1.13.0' ---\nkind: ConfigMap\napiVersion: v1\nmetadata:\nname: ovn-vpc-nat-gw-config\nnamespace: kube-system\ndata:\nenable-vpc-nat-gw: 'true'\n</code></pre> <ul> <li><code>image</code>: \u7f51\u5173 Pod \u6240\u4f7f\u7528\u7684\u955c\u50cf\u3002</li> <li><code>enable-vpc-nat-gw</code>\uff1a \u63a7\u5236\u662f\u5426\u542f\u7528 VPC \u7f51\u5173\u529f\u80fd\u3002</li> </ul>"},{"location":"guide/vpc/#vpc_4","title":"\u521b\u5efa VPC \u7f51\u5173\u5e76\u914d\u7f6e\u9ed8\u8ba4\u8def\u7531","text":"<pre><code>kind: VpcNatGateway\napiVersion: kubeovn.io/v1\nmetadata:\nname: gw1\nspec:\nvpc: test-vpc-1\nsubnet: net1\nlanIp: 10.0.1.254\nselector:\n- \"kubernetes.io/hostname: kube-ovn-worker\"\n- \"kubernetes.io/os: linux\"\nexternalSubnets:\n- ovn-vpc-external-network\n</code></pre> <ul> <li><code>vpc</code>\uff1a\u8be5 VpcNatGateway \u6240\u5c5e\u7684 VPC\u3002</li> <li><code>subnet</code>\uff1a\u4e3a VPC \u5185\u67d0\u4e2a Subnet \u540d\uff0cVPC \u7f51\u5173 Pod \u4f1a\u5728\u8be5\u5b50\u7f51\u4e0b\u7528 <code>lanIp</code> \u6765\u8fde\u63a5\u79df\u6237\u7f51\u7edc\u3002</li> <li><code>lanIp</code>\uff1a<code>subnet</code> \u5185\u67d0\u4e2a\u672a\u88ab\u4f7f\u7528\u7684 IP\uff0cVPC \u7f51\u5173 Pod \u6700\u7ec8\u4f1a\u4f7f\u7528\u8be5 Pod\u3002\u5f53 VPC \u914d\u7f6e\u8def\u7531\u9700\u8981\u6307\u5411\u5f53\u524d VpcNatGateway \u65f6 <code>nextHopIP</code> \u9700\u8981\u8bbe\u7f6e\u4e3a\u8fd9\u4e2a <code>lanIp</code>\u3002</li> <li><code>selector</code>\uff1aVpcNatGateway Pod \u7684\u8282\u70b9\u9009\u62e9\u5668\uff0c\u683c\u5f0f\u548c Kubernetes \u4e2d\u7684 NodeSelector \u683c\u5f0f\u76f8\u540c\u3002</li> <li><code>externalSubnets</code>\uff1a VPC \u7f51\u5173\u4f7f\u7528\u7684\u5916\u90e8\u7f51\u7edc\uff0c\u5982\u679c\u4e0d\u914d\u7f6e\u5219\u9ed8\u8ba4\u4f7f\u7528 <code>ovn-vpc-external-network</code>\uff0c\u5f53\u524d\u7248\u672c\u53ea\u652f\u6301\u914d\u7f6e\u4e00\u4e2a\u5916\u90e8\u7f51\u7edc\u3002</li> </ul> <p>\u5176\u4ed6\u53ef\u914d\u53c2\u6570\uff1a</p> <ul> <li><code>tolerations</code> : \u4e3a VPC \u7f51\u5173\u914d\u7f6e\u5bb9\u5fcd\u5ea6\uff0c\u5177\u4f53\u914d\u7f6e\u53c2\u8003 \u6c61\u70b9\u548c\u5bb9\u5fcd\u5ea6\u3002</li> <li><code>affinity</code> :  \u4e3a VPC \u7f51\u5173 Pod \u6216\u8282\u70b9\u914d\u7f6e\u4eb2\u548c\u6027\uff0c\u5177\u4f53\u8bbe\u7f6e\u53c2\u8003 \u4eb2\u548c\u6027\u4e0e\u53cd\u4eb2\u548c\u6027\u3002</li> </ul>"},{"location":"guide/vpc/#eip","title":"\u521b\u5efa EIP","text":"<p>EIP \u4e3a\u5916\u90e8\u7f51\u7edc\u6bb5\u7684\u67d0\u4e2a IP \u5206\u914d\u7ed9 VPC \u7f51\u5173\u540e\u53ef\u8fdb\u884c DNAT\uff0cSNAT \u548c\u6d6e\u52a8 IP \u64cd\u4f5c\u3002</p> <p>\u968f\u673a\u5206\u914d\u4e00\u4e2a\u5730\u5740\u7ed9 EIP\uff1a</p> <pre><code>kind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-random\nspec:\nnatGwDp: gw1\n</code></pre> <p>\u56fa\u5b9a EIP \u5730\u5740\u5206\u914d\uff1a</p> <pre><code>kind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-static\nspec:\nnatGwDp: gw1\nv4ip: 10.0.1.111\n</code></pre> <p>\u6307\u5b9a EIP \u6240\u5728\u7684\u5916\u90e8\u7f51\u7edc\uff1a</p> <pre><code>kind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-random\nspec:\nnatGwDp: gw1\nexternalSubnet: ovn-vpc-external-network\n</code></pre> <ul> <li><code>externalSubnet</code>\uff1a EIP \u6240\u5728\u5916\u90e8\u7f51\u7edc\u540d\u79f0\uff0c\u5982\u679c\u4e0d\u6307\u5b9a\u5219\u9ed8\u8ba4\u4e3a <code>ovn-vpc-external-network</code>\uff0c\u5982\u679c\u6307\u5b9a\u5219\u5fc5\u987b\u4e3a\u6240\u5728 VPC \u7f51\u5173\u7684 <code>externalSubnets</code> \u4e2d\u7684\u4e00\u4e2a\u3002</li> </ul>"},{"location":"guide/vpc/#dnat","title":"\u521b\u5efa DNAT \u89c4\u5219","text":"<p>\u901a\u8fc7 DNAT \u89c4\u5219\uff0c\u5916\u90e8\u53ef\u4ee5\u901a\u8fc7\u4e00\u4e2a EIP \u52a0\u7aef\u53e3\u7684\u65b9\u5f0f\u6765\u8bbf\u95ee VPC \u5185\u7684\u4e00\u4e2a IP \u548c\u7aef\u53e3\u3002</p> <pre><code>kind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eipd01\nspec:\nnatGwDp: gw1\n\n---\nkind: IptablesDnatRule\napiVersion: kubeovn.io/v1\nmetadata:\nname: dnat01\nspec:\neip: eipd01 externalPort: '8888'\ninternalIp: 10.0.1.10\ninternalPort: '80'\nprotocol: tcp\n</code></pre>"},{"location":"guide/vpc/#snat","title":"\u521b\u5efa SNAT \u89c4\u5219","text":"<p>\u901a\u8fc7 SNAT \u89c4\u5219\uff0cVPC \u5185\u7684 Pod \u8bbf\u95ee\u5916\u90e8\u7684\u5730\u5740\u65f6\u5c06\u4f1a\u901a\u8fc7\u5bf9\u5e94 EIP \u8fdb\u884c SNAT\u3002</p> <pre><code>---\nkind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eips01\nspec:\nnatGwDp: gw1\n---\nkind: IptablesSnatRule\napiVersion: kubeovn.io/v1\nmetadata:\nname: snat01\nspec:\neip: eips01\ninternalCIDR: 10.0.1.0/24\n</code></pre>"},{"location":"guide/vpc/#ip","title":"\u521b\u5efa\u6d6e\u52a8 IP","text":"<p>\u901a\u8fc7\u6d6e\u52a8 IP \u89c4\u5219\uff0cVPC \u5185\u7684\u4e00\u4e2a IP \u4f1a\u548c EIP \u8fdb\u884c\u5b8c\u5168\u6620\u5c04\uff0c\u5916\u90e8\u53ef\u4ee5\u901a\u8fc7\u8fd9\u4e2a EIP \u65b9\u4f4d VPC \u5185\u7684 IP\uff0cVPC \u5185\u7684\u8fd9\u4e2a IP \u8bbf\u95ee\u5916\u90e8\u5730\u5740\u65f6\u4e5f\u4f1a SNAT \u6210\u8fd9\u4e2a EIP\u3002</p> <pre><code>---\nkind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eipf01\nspec:\nnatGwDp: gw1\n\n---\nkind: IptablesFIPRule\napiVersion: kubeovn.io/v1\nmetadata:\nname: fip01\nspec:\neip: eipf01\ninternalIp: 10.0.1.5\n</code></pre>"},{"location":"guide/vpc/#_2","title":"\u81ea\u5b9a\u4e49\u8def\u7531","text":"<p>\u5728\u81ea\u5b9a\u4e49 VPC \u5185\uff0c\u7528\u6237\u53ef\u4ee5\u81ea\u5b9a\u4e49\u7f51\u7edc\u5185\u90e8\u7684\u8def\u7531\u89c4\u5219\uff0c\u7ed3\u5408\u7f51\u5173\u5b9e\u73b0\u66f4\u7075\u6d3b\u7684\u8f6c\u53d1\u3002 Kube-OVN \u652f\u6301\u9759\u6001\u8def\u7531\u548c\u66f4\u4e3a\u7075\u6d3b\u7684\u7b56\u7565\u8def\u7531\u3002</p>"},{"location":"guide/vpc/#_3","title":"\u9759\u6001\u8def\u7531","text":"<pre><code>kind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: test-vpc-1\nspec:\nstaticRoutes:\n- cidr: 0.0.0.0/0\nnextHopIP: 10.0.1.254\npolicy: policyDst\n- cidr: 172.31.0.0/24\nnextHopIP: 10.0.1.253\npolicy: policySrc\nrouteTable: \"rtb1\"\n</code></pre> <ul> <li><code>policy</code>: \u652f\u6301\u76ee\u7684\u5730\u5740\u8def\u7531 <code>policyDst</code> \u548c\u6e90\u5730\u5740\u8def\u7531 <code>policySrc</code>\u3002</li> <li>\u5f53\u8def\u7531\u89c4\u5219\u5b58\u5728\u91cd\u53e0\u65f6\uff0cCIDR \u63a9\u7801\u8f83\u957f\u7684\u89c4\u5219\u4f18\u5148\u7ea7\u66f4\u9ad8\uff0c\u82e5\u63a9\u7801\u957f\u5ea6\u76f8\u540c\u5219\u76ee\u7684\u5730\u5740\u8def\u7531\u4f18\u5148\u4e8e\u6e90\u5730\u5740\u8def\u7531\u3002</li> <li><code>routeTable</code>: \u53ef\u6307\u5b9a\u9759\u6001\u8def\u7531\u6240\u5728\u7684\u8def\u7531\u8868\uff0c\u9ed8\u8ba4\u5728\u4e3b\u8def\u7531\u8868\u3002\u5b50\u7f51\u5173\u8054\u8def\u7531\u8868\u8bf7\u53c2\u8003\u521b\u5efa\u5b50\u7f51</li> </ul>"},{"location":"guide/vpc/#_4","title":"\u7b56\u7565\u8def\u7531","text":"<p>\u9488\u5bf9\u9759\u6001\u8def\u7531\u5339\u914d\u7684\u6d41\u91cf\uff0c\u53ef\u901a\u8fc7\u7b56\u7565\u8def\u7531\u8fdb\u884c\u66f4\u7ec6\u7c92\u5ea6\u7684\u63a7\u5236\u3002\u7b56\u7565\u8def\u7531\u63d0\u4f9b\u4e86\u66f4\u7cbe\u786e\u7684\u5339\u914d\u89c4\u5219\uff0c\u4f18\u5148\u7ea7\u63a7\u5236 \u548c\u66f4\u591a\u7684\u8f6c\u53d1\u52a8\u4f5c\u3002\u8be5\u529f\u80fd\u4e3a OVN \u5185\u90e8\u903b\u8f91\u8def\u7531\u5668\u7b56\u7565\u529f\u80fd\u7684\u4e00\u4e2a\u5bf9\u5916\u66b4\u9732\uff0c\u66f4\u591a\u4f7f\u7528\u4fe1\u606f\u8bf7\u53c2\u8003 Logical Router Policy\u3002</p> <p>\u7b80\u5355\u793a\u4f8b\u5982\u4e0b\uff1a</p> <pre><code>kind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: test-vpc-1\nspec:\npolicyRoutes:\n- action: drop\nmatch: ip4.src==10.0.1.0/24 &amp;&amp; ip4.dst==10.0.1.250\npriority: 11\n- action: reroute\nmatch: ip4.src==10.0.1.0/24\nnextHopIP: 10.0.1.252\npriority: 10\n</code></pre>"},{"location":"guide/vpc/#_5","title":"\u81ea\u5b9a\u4e49\u5185\u90e8\u8d1f\u8f7d\u5747\u8861\u89c4\u5219","text":"<p>Kubernetes \u672c\u8eab\u63d0\u4f9b\u7684 Service \u80fd\u529b\u53ef\u4ee5\u5b8c\u6210\u5185\u90e8\u8d1f\u8f7d\u5747\u8861\u7684\u529f\u80fd\uff0c\u4f46\u662f\u53d7\u9650\u4e8e Kubernetes \u5b9e\u73b0\uff0c Service \u7684 IP \u5730\u5740\u662f\u5168\u5c40\u5206\u914d\u4e14\u4e0d\u80fd\u91cd\u590d\u3002\u5bf9\u4e8e VPC \u7684\u4f7f\u7528\u573a\u666f\uff0c\u7528\u6237\u5e0c\u671b\u80fd\u81ea\u5b9a\u4e49\u5185\u90e8\u8d1f\u8f7d\u5747\u8861\u7684\u5730\u5740 \u8303\u56f4\uff0c\u4e0d\u540c VPC \u4e0b\u7684\u8d1f\u8f7d\u5747\u8861\u5730\u5740\u53ef\u80fd\u91cd\u53e0\uff0cKubernetes \u5185\u7f6e\u7684 Service \u529f\u80fd\u65e0\u6cd5\u5b8c\u5168\u6ee1\u8db3\u3002</p> <p>\u9488\u5bf9\u8fd9\u7c7b\u573a\u666f\uff0cKube-OVN \u63d0\u4f9b\u4e86 <code>SwitchLBRule</code> \u8d44\u6e90\uff0c\u7528\u6237\u53ef\u4ee5\u81ea\u5b9a\u4e49\u5185\u90e8\u8d1f\u8f7d\u5747\u8861\u7684\u5730\u5740\u8303\u56f4\u3002</p> <p>\u4e00\u4e2a `SwitchLBRule`` \u4f8b\u5b50\u5982\u4e0b\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: SwitchLBRule\nmetadata:\nname:  cjh-slr-nginx\nspec:\nvip: 1.1.1.1\nsessionAffinity: ClientIP\nnamespace: default\nselector:\n- app:nginx\nports:\n- name: dns\nport: 8888\ntargetPort: 80\nprotocol: TCP\n</code></pre> <ul> <li><code>vip</code>\uff1a\u81ea\u5b9a\u4e49\u5185\u90e8\u8d1f\u8f7d\u5747\u8861\u7684\u5730\u5740\u3002</li> <li><code>namespace</code>\uff1a\u8d1f\u8f7d\u5747\u8861\u5668\u540e\u7aef Pod \u6240\u5728\u7684 Namespace\u3002</li> <li><code>sessionAffinity</code>\uff1a\u548c Service \u7684 <code>sessionAffinity</code> \u529f\u80fd\u76f8\u540c\u3002</li> <li><code>selector</code>\uff1a\u548c Service \u7684 <code>selector</code> \u529f\u80fd\u76f8\u540c\u3002</li> <li><code>ports</code>\uff1a\u548c Service \u7684 <code>port</code> \u529f\u80fd\u76f8\u540c\u3002</li> </ul> <p>\u67e5\u770b\u90e8\u7f72\u7684\u5185\u90e8\u8d1f\u8f7d\u5747\u8861\u5668\u89c4\u5219\uff1a</p> <pre><code># kubectl get slr\nNAME                VIP         PORT(S)                  SERVICE                             AGE\nvpc-dns-test-cjh2   10.96.0.3   53/UDP,53/TCP,9153/TCP   kube-system/slr-vpc-dns-test-cjh2   88m\n</code></pre>"},{"location":"guide/vpc/#vpc-dns","title":"\u81ea\u5b9a\u4e49 vpc-dns","text":"<p>\u7531\u4e8e\u81ea\u5b9a\u4e49 VPC \u548c\u9ed8\u8ba4 VPC \u7f51\u7edc\u76f8\u4e92\u9694\u79bb\uff0cVPC \u5185 Pod \u65e0\u6cd5\u4f7f\u7528\u9ed8\u8ba4\u7684 coredns \u670d\u52a1\u8fdb\u884c\u57df\u540d\u89e3\u6790\u3002 \u5982\u679c\u5e0c\u671b\u5728\u81ea\u5b9a\u4e49 VPC \u5185\u4f7f\u7528 coredns \u89e3\u6790\u96c6\u7fa4\u5185 Service \u57df\u540d\uff0c\u53ef\u4ee5\u901a\u8fc7 Kube-OVN \u63d0\u4f9b\u7684 vpc-dns \u8d44\u6e90\u6765\u5b9e\u73b0\u3002</p>"},{"location":"guide/vpc/#_6","title":"\u521b\u5efa\u9644\u52a0\u7f51\u5361","text":"<pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: ovn-nad\nnamespace: default\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.0\",\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"ovn-nad.default.ovn\"\n}'\n</code></pre>"},{"location":"guide/vpc/#ovn-default-provider","title":"\u4fee\u6539 ovn-default \u903b\u8f91\u4ea4\u6362\u673a\u7684 provider","text":"<p>\u4fee\u6539 ovn-default \u7684 provider\uff0c\u4e3a\u4e0a\u9762 nad \u914d\u7f6e\u7684 provider <code>ovn-nad.default.ovn</code>\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: ovn-default\nspec:\ncidrBlock: 10.16.0.0/16\ndefault: true\ndisableGatewayCheck: false\ndisableInterConnection: false\nenableDHCP: false\nenableIPv6RA: false\nexcludeIps:\n- 10.16.0.1\ngateway: 10.16.0.1\ngatewayType: distributed\nlogicalGateway: false\nnatOutgoing: true\nprivate: false\nprotocol: IPv4\nprovider: ovn-nad.default.ovn\nvpc: ovn-cluster\n</code></pre>"},{"location":"guide/vpc/#vpc-dns-configmap","title":"\u914d\u7f6e vpc-dns \u7684 ConfigMap","text":"<p>\u5728 kube-system \u547d\u540d\u7a7a\u95f4\u4e0b\u521b\u5efa configmap\uff0c\u914d\u7f6e vpc-dns \u4f7f\u7528\u53c2\u6570\uff0c\u7528\u4e8e\u540e\u9762\u542f\u52a8 vpc-dns \u529f\u80fd\uff1a</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: vpc-dns-config\nnamespace: kube-system\ndata:\ncoredns-vip: 10.96.0.3\nenable-vpc-dns: true\nnad-name: ovn-nad\nnad-provider: ovn-nad.default.ovn\n</code></pre> <ul> <li><code>enable-vpc-dns</code>\uff1a\uff08\u53ef\u7f3a\u7701\uff09<code>true</code> \u542f\u7528\u529f\u80fd\uff0c<code>false</code> \u5173\u95ed\u529f\u80fd\u3002\u9ed8\u8ba4 <code>true</code>\u3002</li> <li><code>coredns-image</code>\uff1a\uff08\u53ef\u7701\u7565\uff09\uff1adns \u90e8\u7f72\u955c\u50cf\u3002\u9ed8\u8ba4\u4e3a\u96c6\u7fa4 coredns \u90e8\u7f72\u7248\u672c\u3002</li> <li><code>coredns-template</code>\uff1a\uff08\u53ef\u7701\u7565\uff09\uff1adns \u90e8\u7f72\u6a21\u677f\u6240\u5728\u7684 URL\u3002\u9ed8\u8ba4\uff1a\u5f53\u524d\u7248\u672c\u4ed3\u5e93\u91cc\u7684 <code>yamls/coredns-template.yaml</code>\u3002</li> <li><code>coredns-vip</code>\uff1a\u4e3a coredns \u63d0\u4f9b lb \u670d\u52a1\u7684 vip\u3002</li> <li><code>nad-name</code>\uff1a\u914d\u7f6e\u7684 <code>network-attachment-definitions</code> \u8d44\u6e90\u540d\u79f0\u3002</li> <li><code>nad-provider</code>\uff1a\u4f7f\u7528\u7684 provider \u540d\u79f0\u3002</li> <li><code>k8s-service-host</code>\uff1a\uff08\u53ef\u7f3a\u7701\uff09 \u7528\u4e8e coredns \u8bbf\u95ee k8s apiserver \u670d\u52a1\u7684 ip\u3002</li> <li><code>k8s-service-port</code>\uff1a\uff08\u53ef\u7f3a\u7701\uff09\u7528\u4e8e coredns \u8bbf\u95ee k8s apiserver \u670d\u52a1\u7684 port\u3002</li> </ul>"},{"location":"guide/vpc/#vpc-dns_1","title":"\u90e8\u7f72 vpc-dns \u4f9d\u8d56\u8d44\u6e90","text":"<pre><code>apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\nlabels:\nkubernetes.io/bootstrapping: rbac-defaults\nname: system:vpc-dns\nrules:\n- apiGroups:\n- \"\"\nresources:\n- endpoints\n- services\n- pods\n- namespaces\nverbs:\n- list\n- watch\n- apiGroups:\n- discovery.k8s.io\nresources:\n- endpointslices\nverbs:\n- list\n- watch\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\nannotations:\nrbac.authorization.kubernetes.io/autoupdate: \"true\"\nlabels:\nkubernetes.io/bootstrapping: rbac-defaults\nname: vpc-dns\nroleRef:\napiGroup: rbac.authorization.k8s.io\nkind: ClusterRole\nname: system:vpc-dns\nsubjects:\n- kind: ServiceAccount\nname: vpc-dns\nnamespace: kube-system\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\nname: vpc-dns\nnamespace: kube-system\n---\napiVersion: v1\nkind: ConfigMap\nmetadata:\nname: vpc-dns-corefile\nnamespace: kube-system\ndata:\nCorefile: |\n.:53 {\nerrors\nhealth {\nlameduck 5s\n}\nready\nkubernetes cluster.local in-addr.arpa ip6.arpa {\npods insecure\nfallthrough in-addr.arpa ip6.arpa\n}\nprometheus :9153\nforward . /etc/resolv.conf {\nprefer_udp\n}\ncache 30\nloop\nreload\nloadbalance\n}\n</code></pre>"},{"location":"guide/vpc/#vpc-dns_2","title":"\u90e8\u7f72 vpc-dns","text":"<pre><code>kind: VpcDns\napiVersion: kubeovn.io/v1\nmetadata:\nname: test-cjh1\nspec:\nvpc: cjh-vpc-1\nsubnet: cjh-subnet-1\n</code></pre> <ul> <li><code>vpc</code>\uff1a \u7528\u4e8e\u90e8\u7f72 dns \u7ec4\u4ef6\u7684 vpc \u540d\u79f0\u3002</li> <li><code>subnet</code>\uff1a\u7528\u4e8e\u90e8\u7f72 dns \u7ec4\u4ef6\u7684\u5b50\u540d\u79f0\u3002</li> </ul> <p>\u67e5\u770b\u8d44\u6e90\u4fe1\u606f\uff1a</p> <pre><code>[root@hci-dev-mst-1 kubeovn]# kubectl get vpc-dns\nNAME        ACTIVE   VPC         SUBNET   \ntest-cjh1   false    cjh-vpc-1   cjh-subnet-1   \ntest-cjh2   true     cjh-vpc-1   cjh-subnet-2 </code></pre> <ul> <li><code>ACTIVE</code>: <code>true</code> \u6210\u529f\u90e8\u7f72\u4e86\u81ea\u5b9a\u4e49 dns \u7ec4\u4ef6\uff0c<code>false</code> \u65e0\u90e8\u7f72</li> </ul>"},{"location":"guide/vpc/#_7","title":"\u9650\u5236","text":"<ul> <li>\u4e00\u4e2a vpc \u4e0b\u53ea\u4f1a\u90e8\u7f72\u4e00\u4e2a\u81ea\u5b9a\u4e49 dns \u7ec4\u4ef6;</li> <li>\u5f53\u4e00\u4e2a vpc \u4e0b\u914d\u7f6e\u591a\u4e2a vpc-dns \u8d44\u6e90\uff08\u5373\u540c\u4e00\u4e2a vpc \u4e0d\u540c\u7684 subnet\uff09\uff0c\u53ea\u6709\u4e00\u4e2a vpc-dns \u8d44\u6e90\u72b6\u6001 <code>true</code>\uff0c\u5176\u4ed6\u4e3a <code>fasle</code>;</li> <li>\u5f53 <code>true</code> \u7684 vpc-dns \u88ab\u5220\u9664\u6389\uff0c\u4f1a\u83b7\u53d6\u5176\u4ed6 <code>false</code> \u7684 vpc-dns \u8fdb\u884c\u90e8\u7f72\u3002</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/webhook/","title":"Webhook \u4f7f\u7528","text":"<p>\u4f7f\u7528 Webhook \u53ef\u4ee5\u5bf9 Kube-OVN \u5185\u7684 CRD \u8d44\u6e90\u8fdb\u884c\u6821\u9a8c\uff0c\u76ee\u524d Webhook \u4e3b\u8981\u5b8c\u6210 \u56fa\u5b9a IP \u5730\u5740\u51b2\u7a81\u68c0\u6d4b\u548c Subnet CIDR \u7684\u51b2\u7a81\u68c0\u6d4b\uff0c\u5e76\u5728\u8fd9\u7c7b\u8d44\u6e90\u521b\u5efa\u51b2\u7a81\u65f6\u63d0\u793a\u9519\u8bef\u3002</p> <p>\u7531\u4e8e Webhook \u4f1a\u62e6\u622a\u6240\u6709\u7684 Subnet \u548c Pod \u521b\u5efa\u7684\u8bf7\u6c42\uff0c\u56e0\u6b64\u9700\u8981\u5148\u90e8\u7f72 Kube-OVN \u540e\u90e8\u7f72 Webhook \u907f\u514d\u65e0\u6cd5\u521b\u5efa Pod\u3002</p>"},{"location":"guide/webhook/#cert-manager","title":"Cert-Manager \u5b89\u88c5","text":"<p>Webhook \u90e8\u7f72\u9700\u8981\u76f8\u5173\u8bc1\u4e66\u52a0\u5bc6\uff0c\u6211\u4eec\u4f7f\u7528 cert-manager \u751f\u6210\u76f8\u5173\u8bc1\u4e66\uff0c\u6211\u4eec\u9700\u8981\u5728\u90e8\u7f72 Webhook \u524d\u5148\u90e8\u7f72 cert-manager\u3002</p> <p>\u53ef\u4ee5\u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u90e8\u7f72 cert-manager:</p> <pre><code>kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.8.0/cert-manager.yaml\n</code></pre> <p>\u66f4\u591a cert-manager \u4f7f\u7528\u8bf7\u53c2\u8003 cert-manager \u6587\u6863\u3002</p>"},{"location":"guide/webhook/#webhook_1","title":"\u5b89\u88c5 Webhook","text":"<p>\u4e0b\u8f7d Webhook \u5bf9\u5e94\u7684 yaml \u8fdb\u884c\u5b89\u88c5:</p> <pre><code># kubectl apply -f https://raw.githubusercontent.com/kubeovn/kube-ovn/master/yamls/webhook.yaml\ndeployment.apps/kube-ovn-webhook created\nservice/kube-ovn-webhook created\nvalidatingwebhookconfiguration.admissionregistration.k8s.io/kube-ovn-webhook created\ncertificate.cert-manager.io/kube-ovn-webhook-serving-cert created\nissuer.cert-manager.io/kube-ovn-webhook-selfsigned-issuer created\n</code></pre>"},{"location":"guide/webhook/#webhook_2","title":"\u9a8c\u8bc1 Webhook \u751f\u6548","text":"<p>\u67e5\u770b\u5df2\u8fd0\u884c Pod\uff0c\u5f97\u5230 Pod IP <code>10.16.0.15</code>\uff1a</p> <pre><code># kubectl get pod -o wide\nNAME                      READY   STATUS    RESTARTS   AGE     IP           NODE              NOMINATED NODE   READINESS GATES\nstatic-7584848b74-fw9dm   1/1     Running   0          2d13h   10.16.0.15   kube-ovn-worker   &lt;none&gt; </code></pre> <p>\u7f16\u5199 yaml \u521b\u5efa\u76f8\u540c IP \u7684 Pod\uff1a</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nannotations:\novn.kubernetes.io/ip_address: 10.16.0.15\novn.kubernetes.io/mac_address: 00:00:00:53:6B:B6\nlabels:\napp: static\nmanagedFields:\nname: staticip-pod\nnamespace: default\nspec:\ncontainers:\n- image: docker.io/library/nginx:alpine\nimagePullPolicy: IfNotPresent\nname: qatest\n</code></pre> <p>\u4f7f\u7528\u4ee5\u4e0a yaml \u521b\u5efa\u9759\u6001\u5730\u5740 Pod \u7684\u65f6\u5019\uff0c\u63d0\u793a IP \u5730\u5740\u51b2\u7a81\uff1a</p> <pre><code># kubectl apply -f pod-static.yaml\nError from server (annotation ip address 10.16.0.15 is conflict with ip crd static-7584848b74-fw9dm.default 10.16.0.15): error when creating \"pod-static.yaml\": admission webhook \"pod-ip-validaing.kube-ovn.io\" denied the request: annotation ip address 10.16.0.15 is conflict with ip crd static-7584848b74-fw9dm.default 10.16.0.15\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"ops/change-default-subnet/","title":"\u4fee\u6539\u5b50\u7f51 CIDR","text":"<p>\u5982\u679c\u521b\u5efa\u7684\u5b50\u7f51 CIDR \u51b2\u7a81\u6216\u4e0d\u7b26\u5408\u9884\u671f\uff0c\u53ef\u4ee5\u901a\u8fc7\u672c\u6587\u6863\u7684\u6b65\u9aa4\u8fdb\u884c\u4fee\u6539\u3002</p> <p>\u4fee\u6539\u5b50\u7f51 CIDR \u540e\u4e4b\u524d\u521b\u5efa\u7684 Pod \u5c06\u65e0\u6cd5\u6b63\u5e38\u8bbf\u95ee\u7f51\u7edc\u9700\u8981\u8fdb\u884c\u91cd\u5efa\u3002 \u5efa\u8bae\u64cd\u4f5c\u524d\u614e\u91cd\u8003\u8651\u3002\u672c\u6587\u53ea\u9488\u5bf9\u4e1a\u52a1\u5b50\u7f51 CIDR \u66f4\u6539\u8fdb\u884c\u64cd\u4f5c\uff0c\u5982\u9700 \u66f4\u6539 Join \u5b50\u7f51 CIDR \u8bf7\u53c2\u8003\u66f4\u6539 Join \u5b50\u7f51 CIDR\u3002</p>"},{"location":"ops/change-default-subnet/#_1","title":"\u7f16\u8f91\u5b50\u7f51","text":"<p>\u4f7f\u7528 <code>kubectl edit</code> \u4fee\u6539\u5b50\u7f51 <code>cidrBlock</code>\uff0c<code>gateway</code> \u548c <code>excludeIps</code>\u3002</p> <pre><code>kubectl edit subnet test-subnet\n</code></pre>"},{"location":"ops/change-default-subnet/#namespace-pod","title":"\u91cd\u5efa\u8be5\u5b50\u7f51\u7ed1\u5b9a\u7684 Namespace \u4e0b\u6240\u6709 Pod","text":"<p>\u4ee5\u5b50\u7f51\u7ed1\u5b9a <code>test</code> Namespace \u4e3a\u4f8b\uff1a</p> <pre><code>for pod in $(kubectl get pod --no-headers -n \"$ns\" --field-selector spec.restartPolicy=Always -o custom-columns=NAME:.metadata.name,HOST:spec.hostNetwork | awk '{if ($2!=\"true\") print $1}'); do\nkubectl delete pod \"$pod\" -n test --ignore-not-found\ndone\n</code></pre> <p>\u82e5\u53ea\u4f7f\u7528\u4e86\u9ed8\u8ba4\u5b50\u7f51\uff0c\u53ef\u4ee5\u4f7f\u7528\u4e0b\u5217\u547d\u4ee4\u5220\u9664\u6240\u6709\u975e host \u7f51\u7edc\u6a21\u5f0f\u7684 Pod\uff1a</p> <pre><code>for ns in $(kubectl get ns --no-headers -o custom-columns=NAME:.metadata.name); do\nfor pod in $(kubectl get pod --no-headers -n \"$ns\" --field-selector spec.restartPolicy=Always -o custom-columns=NAME:.metadata.name,HOST:spec.hostNetwork | awk '{if ($2!=\"true\") print $1}'); do\nkubectl delete pod \"$pod\" -n \"$ns\" --ignore-not-found\n  done\ndone\n</code></pre>"},{"location":"ops/change-default-subnet/#_2","title":"\u66f4\u6539\u9ed8\u8ba4\u5b50\u7f51\u914d\u7f6e","text":"<p>\u82e5\u4fee\u6539\u7684\u4e3a\u9ed8\u8ba4\u5b50\u7f51\u7684 CIDR \u8fd8\u9700\u8981\u66f4\u6539 <code>kube-ovn-controller</code> Deployment \u7684\u542f\u52a8\u53c2\u6570\uff1a</p> <pre><code>args:\n- --default-cidr=10.17.0.0/16\n- --default-gateway=10.17.0.1\n- --default-exclude-ips=10.17.0.1\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"ops/change-join-subnet/","title":"\u4fee\u6539 Join \u5b50\u7f51 CIDR","text":"<p>\u82e5\u53d1\u73b0\u521b\u5efa\u7684 Join \u5b50\u7f51 CIDR \u51b2\u7a81\u6216\u4e0d\u7b26\u5408\u9884\u671f\uff0c\u53ef\u4ee5\u901a\u8fc7\u672c\u6587\u6863\u8fdb\u884c\u4fee\u6539\u3002</p> <p>\u4fee\u6539 Join \u5b50\u7f51 CIDR \u540e\u4e4b\u524d\u521b\u5efa\u7684 Pod \u5c06\u65e0\u6cd5\u6b63\u5e38\u8bbf\u95ee\u5916\u90e8\u7f51\u7edc\uff0c\u9700\u8981\u7b49\u91cd\u5efa\u5b8c\u6210, \u5efa\u8bae\u524d\u64cd\u4f5c\u65f6\u614e\u91cd\u8003\u8651\u3002</p>"},{"location":"ops/change-join-subnet/#join","title":"\u5220\u9664 Join \u5b50\u7f51","text":"<pre><code>kubectl patch subnet join --type='json' -p '[{\"op\": \"replace\", \"path\": \"/metadata/finalizers\", \"value\": []}]'\nkubectl delete subnet join\n</code></pre>"},{"location":"ops/change-join-subnet/#_1","title":"\u6e05\u7406\u76f8\u5173\u5206\u914d\u4fe1\u606f","text":"<pre><code>kubectl annotate node ovn.kubernetes.io/allocated=false --all --overwrite\n</code></pre>"},{"location":"ops/change-join-subnet/#join_1","title":"\u4fee\u6539 Join \u5b50\u7f51\u76f8\u5173\u4fe1\u606f","text":"<p>\u4fee\u6539 <code>kube-ovn-controller</code> \u5185 Join \u5b50\u7f51\u76f8\u5173\u4fe1\u606f\uff1a</p> <pre><code>kubectl edit deployment -n kube-system kube-ovn-controller\n</code></pre> <p>\u4fee\u6539\u4e0b\u5217\u53c2\u6570\uff1a</p> <pre><code>args:\n- --node-switch-cidr=100.51.0.0/16\n</code></pre> <p>\u91cd\u542f <code>kube-ovn-controller</code> \u91cd\u5efa <code>join</code> \u5b50\u7f51\uff1a</p> <pre><code>kubectl delete pod -n kube-system -lapp=kube-ovn-controller\n</code></pre> <p>\u67e5\u770b\u65b0\u7684 Join \u5b50\u7f51\u4fe1\u606f\uff1a</p> <pre><code># kubectl get subnet\nNAME          PROVIDER   VPC           PROTOCOL   CIDR            PRIVATE   NAT     DEFAULT   GATEWAYTYPE   V4USED   V4AVAILABLE   V6USED   V6AVAILABLE   EXCLUDEIPS\njoin          ovn        ovn-cluster   IPv4       100.51.0.0/16   false     false   false     distributed   2        65531         0        0             [\"100.51.0.1\"]\novn-default   ovn        ovn-cluster   IPv4       10.17.0.0/16    false     true    true      distributed   5        65528         0        0             [\"10.17.0.1\"]\n</code></pre>"},{"location":"ops/change-join-subnet/#ovn0","title":"\u91cd\u65b0\u914d\u7f6e ovn0 \u7f51\u5361\u5730\u5740","text":"<p>\u6bcf\u4e2a\u8282\u70b9\u7684 <code>ovn0</code> \u7f51\u5361\u4fe1\u606f\u9700\u8981\u91cd\u65b0\u66f4\u65b0\uff0c\u53ef\u901a\u8fc7\u91cd\u542f <code>kube-ovn-cni</code> \u6765\u5b8c\u6210\uff1a</p> <pre><code>kubectl delete pod -n kube-system -l app=kube-ovn-cni\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"ops/change-log-level/","title":"\u8c03\u6574\u65e5\u5fd7\u7b49\u7ea7","text":"<p>\u6253\u5f00 <code>kube-ovn.yaml</code>\uff0c\u5728\u670d\u52a1\u542f\u52a8\u811a\u672c\u7684\u53c2\u6570\u5217\u8868\u4e2d\u8bbe\u7f6e log \u7b49\u7ea7\uff0c\u6bd4\u5982\uff1a</p> <pre><code>vi kube-ovn.yaml\n# ...\n- name: kube-ovn-controller\n          image: \"docker.io/kubeovn/kube-ovn:v1.13.0\"\nimagePullPolicy: IfNotPresent\n          args:\n          - /kube-ovn/start-controller.sh\n          - --v=3\n# ...\n# log \u7b49\u7ea7\u8d8a\u9ad8\uff0clog \u5c31\u8d8a\u8be6\u7ec6\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"ops/change-ovn-central-node/","title":"\u66f4\u6362 ovn-central \u8282\u70b9","text":"<p>\u7531\u4e8e <code>ovn-central</code> \u5185\u7684 <code>ovn-nb</code> \u548c <code>ovn-sb</code> \u5206\u522b\u5efa\u7acb\u4e86\u7c7b\u4f3c etcd \u7684 raft \u96c6\u7fa4\uff0c\u56e0\u6b64\u66f4\u6362 <code>ovn-central</code> \u8282\u70b9\u9700\u8981\u989d\u5916\u7684\u64cd\u4f5c\uff0c\u4fdd\u8bc1\u96c6\u7fa4\u72b6\u6001\u7684\u6b63\u786e\u548c\u6570\u636e\u7684\u4e00\u81f4\u3002\u5efa\u8bae\u6bcf\u6b21\u53ea\u5bf9\u4e00\u4e2a\u8282\u70b9\u8fdb\u884c\u4e0a\u4e0b\u7ebf\u5904\u7406\uff0c\u4ee5\u907f\u514d\u96c6\u7fa4\u8fdb\u5165\u4e0d\u53ef\u7528 \u72b6\u6001\uff0c\u5f71\u54cd\u96c6\u7fa4\u6574\u4f53\u7f51\u7edc\u3002</p>"},{"location":"ops/change-ovn-central-node/#ovn-central_1","title":"ovn-central \u8282\u70b9\u4e0b\u7ebf","text":"<p>\u672c\u6587\u6863\u9488\u5bf9\u5982\u4e0b\u7684\u96c6\u7fa4\u60c5\u51b5\uff0c\u4ee5\u4e0b\u7ebf <code>kube-ovn-control-plane2</code> \u8282\u70b9\u4e3a\u4f8b\uff0c\u4ecb\u7ecd\u5982\u4f55\u5c06\u5176\u4ece <code>ovn-central</code> \u96c6\u7fa4\u4e2d\u79fb\u9664\u3002</p> <pre><code># kubectl -n kube-system get pod -o wide | grep central\novn-central-6bf58cbc97-2cdhg                      1/1     Running   0             21m   172.18.0.3   kube-ovn-control-plane    &lt;none&gt;           &lt;none&gt;\novn-central-6bf58cbc97-crmfp                      1/1     Running   0             21m   172.18.0.5   kube-ovn-control-plane2   &lt;none&gt;           &lt;none&gt;\novn-central-6bf58cbc97-lxmpl                      1/1     Running   0             21m   172.18.0.4   kube-ovn-control-plane3   &lt;none&gt;           &lt;none&gt;\n</code></pre>"},{"location":"ops/change-ovn-central-node/#ovn-nb","title":"\u4e0b\u7ebf ovn-nb \u96c6\u7fa4\u5185\u5bf9\u5e94\u8282\u70b9","text":"<p>\u9996\u5148\u67e5\u770b\u8282\u70b9\u5728\u96c6\u7fa4\u5185\u7684 ID\uff0c\u4ee5\u4fbf\u540e\u7eed\u64cd\u4f5c\u3002</p> <pre><code># kubectl ko nb status\n1b9a\nName: OVN_Northbound\nCluster ID: 32ca (32ca07fb-739b-4257-b510-12fa18e7cce8)\nServer ID: 1b9a (1b9a5d76-e69b-410c-8085-39943d0cd38c)\nAddress: tcp:[172.18.0.3]:6643\nStatus: cluster member\nRole: leader\nTerm: 1\nLeader: self\nVote: self\n\nLast Election started 2135194 ms ago, reason: timeout\nLast Election won: 2135188 ms ago\nElection timer: 5000\nLog: [135, 135]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: &lt;-d64b -&gt;d64b &lt;-4984 -&gt;4984\nDisconnections: 0\nServers:\n    4984 (4984 at tcp:[172.18.0.4]:6643) next_index=135 match_index=134 last msg 1084 ms ago\n    1b9a (1b9a at tcp:[172.18.0.3]:6643) (self) next_index=2 match_index=134\nd64b (d64b at tcp:[172.18.0.5]:6643) next_index=135 match_index=134 last msg 1084 ms ago\nstatus: ok\n</code></pre> <p><code>kube-ovn-control-plane2</code> \u5bf9\u5e94\u8282\u70b9 IP \u4e3a <code>172.18.0.5</code>\uff0c\u96c6\u7fa4\u5185\u5bf9\u5e94\u7684 ID \u4e3a <code>d64b</code>\u3002\u63a5\u4e0b\u6765\u4ece ovn-nb \u96c6\u7fa4\u4e2d\u8e22\u51fa\u8be5\u8282\u70b9\uff1a</p> <pre><code># kubectl ko nb kick d64b\nstarted removal\n</code></pre> <p>\u786e\u8ba4\u8282\u70b9\u8e22\u51fa\u6210\u529f\uff1a</p> <pre><code># kubectl ko nb status\n1b9a\nName: OVN_Northbound\nCluster ID: 32ca (32ca07fb-739b-4257-b510-12fa18e7cce8)\nServer ID: 1b9a (1b9a5d76-e69b-410c-8085-39943d0cd38c)\nAddress: tcp:[172.18.0.3]:6643\nStatus: cluster member\nRole: leader\nTerm: 1\nLeader: self\nVote: self\n\nLast Election started 2297649 ms ago, reason: timeout\nLast Election won: 2297643 ms ago\nElection timer: 5000\nLog: [136, 136]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: &lt;-4984 -&gt;4984\nDisconnections: 2\nServers:\n    4984 (4984 at tcp:[172.18.0.4]:6643) next_index=136 match_index=135 last msg 1270 ms ago\n    1b9a (1b9a at tcp:[172.18.0.3]:6643) (self) next_index=2 match_index=135\nstatus: ok\n</code></pre>"},{"location":"ops/change-ovn-central-node/#ovn-sb","title":"\u4e0b\u7ebf ovn-sb \u96c6\u7fa4\u5185\u5bf9\u5e94\u8282\u70b9","text":"<p>\u63a5\u4e0b\u6765\u9700\u8981\u64cd\u4f5c ovn-sb \u96c6\u7fa4\uff0c\u9996\u5148\u67e5\u770b\u8282\u70b9\u5728\u96c6\u7fa4\u5185\u7684 ID\uff0c\u4ee5\u4fbf\u540e\u7eed\u64cd\u4f5c\uff1a</p> <pre><code>kubectl ko sb status\n3722\nName: OVN_Southbound\nCluster ID: d4bd (d4bd37a4-0400-499f-b4df-b4fd389780f0)\nServer ID: 3722 (3722d5ae-2ced-4820-a6b2-8b744d11fb3e)\nAddress: tcp:[172.18.0.3]:6644\nStatus: cluster member\nRole: leader\nTerm: 1\nLeader: self\nVote: self\n\nLast Election started 2395317 ms ago, reason: timeout\nLast Election won: 2395316 ms ago\nElection timer: 5000\nLog: [130, 130]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: &lt;-e9f7 -&gt;e9f7 &lt;-6e84 -&gt;6e84\nDisconnections: 0\nServers:\n    e9f7 (e9f7 at tcp:[172.18.0.5]:6644) next_index=130 match_index=129 last msg 1006 ms ago\n    6e84 (6e84 at tcp:[172.18.0.4]:6644) next_index=130 match_index=129 last msg 1004 ms ago\n    3722 (3722 at tcp:[172.18.0.3]:6644) (self) next_index=2 match_index=129\nstatus: ok\n</code></pre> <p><code>kube-ovn-control-plane2</code> \u5bf9\u5e94\u8282\u70b9 IP \u4e3a <code>172.18.0.5</code>\uff0c\u96c6\u7fa4\u5185\u5bf9\u5e94\u7684 ID \u4e3a <code>e9f7</code>\u3002\u63a5\u4e0b\u6765\u4ece ovn-sb \u96c6\u7fa4\u4e2d\u8e22\u51fa\u8be5\u8282\u70b9\uff1a</p> <pre><code># kubectl ko sb kick e9f7\nstarted removal\n</code></pre> <p>\u786e\u8ba4\u8282\u70b9\u8e22\u51fa\u6210\u529f\uff1a</p> <pre><code># kubectl ko sb status\n3722\nName: OVN_Southbound\nCluster ID: d4bd (d4bd37a4-0400-499f-b4df-b4fd389780f0)\nServer ID: 3722 (3722d5ae-2ced-4820-a6b2-8b744d11fb3e)\nAddress: tcp:[172.18.0.3]:6644\nStatus: cluster member\nRole: leader\nTerm: 1\nLeader: self\nVote: self\n\nLast Election started 2481636 ms ago, reason: timeout\nLast Election won: 2481635 ms ago\nElection timer: 5000\nLog: [131, 131]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: &lt;-6e84 -&gt;6e84\nDisconnections: 2\nServers:\n    6e84 (6e84 at tcp:[172.18.0.4]:6644) next_index=131 match_index=130 last msg 642 ms ago\n    3722 (3722 at tcp:[172.18.0.3]:6644) (self) next_index=2 match_index=130\nstatus: ok\n</code></pre>"},{"location":"ops/change-ovn-central-node/#ovn-central_2","title":"\u5220\u9664\u8282\u70b9\u6807\u7b7e\uff0c\u5e76\u7f29\u5bb9 ovn-central","text":"<p>\u6ce8\u610f\u9700\u5728 ovn-central \u73af\u5883\u53d8\u91cf <code>NODE_IPS</code> \u7684\u8282\u70b9\u5730\u5740\u4e2d\u5220\u9664\u4e0b\u7ebf\u8282\u70b9\u3002</p> <pre><code>kubectl label node kube-ovn-control-plane2 kube-ovn/role-\nkubectl scale deployment -n kube-system ovn-central --replicas=2\nkubectl set env deployment/ovn-central -n kube-system NODE_IPS=\"172.18.0.3,172.18.0.4\"\nkubectl rollout status deployment/ovn-central -n kube-system </code></pre>"},{"location":"ops/change-ovn-central-node/#ovn-central_3","title":"\u4fee\u6539\u5176\u4ed6\u7ec4\u4ef6\u8fde\u63a5 ovn-central \u5730\u5740","text":"<p>\u4fee\u6539 <code>ovs-ovn</code> \u5185\u8fde\u63a5\u4fe1\u606f\uff0c\u5220\u9664\u4e0b\u7ebf\u8282\u70b9\u5730\u5740\u3002</p> <pre><code># kubectl set env daemonset/ovs-ovn -n kube-system OVN_DB_IPS=\"172.18.0.3,172.18.0.4\"\ndaemonset.apps/ovs-ovn env updated\n# kubectl delete pod -n kube-system -lapp=ovs\npod \"ovs-ovn-4f6jc\" deleted\npod \"ovs-ovn-csn2w\" deleted\npod \"ovs-ovn-mpbmb\" deleted\n</code></pre> <p>\u4fee\u6539 <code>kube-ovn-controller</code> \u5185\u8fde\u63a5\u4fe1\u606f\uff0c\u5220\u9664\u4e0b\u7ebf\u8282\u70b9\u5730\u5740\u3002</p> <pre><code># kubectl set env deployment/kube-ovn-controller -n kube-system OVN_DB_IPS=\"172.18.0.3,172.18.0.4\"\ndeployment.apps/kube-ovn-controller env updated\n\n# kubectl rollout status deployment/kube-ovn-controller -n kube-system\nWaiting for deployment \"kube-ovn-controller\" rollout to finish: 1 of 3 updated replicas are available...\nWaiting for deployment \"kube-ovn-controller\" rollout to finish: 2 of 3 updated replicas are available...\ndeployment \"kube-ovn-controller\" successfully rolled out\n</code></pre>"},{"location":"ops/change-ovn-central-node/#_1","title":"\u6e05\u7406\u8282\u70b9","text":"<p>\u5220\u9664 <code>kube-ovn-control-plane2</code> \u8282\u70b9\u5185\u7684\u6570\u636e\u5e93\u6587\u4ef6\uff0c\u907f\u514d\u91cd\u590d\u6dfb\u52a0\u8282\u70b9\u65f6\u53d1\u751f\u5f02\u5e38\uff1a</p> <pre><code>rm -rf /etc/origin/ovn\n</code></pre> <p>\u5982\u9700\u5c06\u8282\u70b9\u4ece\u6574\u4e2a Kubernetes \u96c6\u7fa4\u4e0b\u7ebf\uff0c\u8fd8\u9700\u7ee7\u7eed\u53c2\u8003\u5220\u9664\u5de5\u4f5c\u8282\u70b9\u8fdb\u884c\u64cd\u4f5c\u3002</p>"},{"location":"ops/change-ovn-central-node/#ovn-central_4","title":"ovn-central \u8282\u70b9\u4e0a\u7ebf","text":"<p>\u4e0b\u5217\u6b65\u9aa4\u4f1a\u5c06\u4e00\u4e2a\u65b0\u7684 Kubernetes \u8282\u70b9\u52a0\u5165 <code>ovn-central</code> \u96c6\u7fa4\u3002</p>"},{"location":"ops/change-ovn-central-node/#_2","title":"\u76ee\u5f55\u68c0\u67e5","text":"<p>\u68c0\u67e5\u65b0\u589e\u8282\u70b9\u7684 <code>/etc/origin/ovn</code> \u76ee\u5f55\u4e2d\u662f\u5426\u5b58\u5728 <code>ovnnb_db.db</code> \u6216 <code>ovnsb_db.db</code> \u6587\u4ef6\uff0c\u82e5\u5b58\u5728\u9700\u63d0\u524d\u5220\u9664\uff1a</p> <pre><code>rm -rf /etc/origin/ovn\n</code></pre>"},{"location":"ops/change-ovn-central-node/#ovn-central_5","title":"\u786e\u8ba4\u5f53\u524d ovn-central \u96c6\u7fa4\u72b6\u6001\u6b63\u5e38","text":"<p>\u82e5\u5f53\u524d <code>ovn-central</code> \u96c6\u7fa4\u72b6\u6001\u5df2\u7ecf\u5f02\u5e38\uff0c\u65b0\u589e\u8282\u70b9\u53ef\u80fd\u5bfc\u81f4\u6295\u7968\u9009\u4e3e\u65e0\u6cd5\u8fc7\u534a\u6570\uff0c\u5f71\u54cd\u540e\u7eed\u64cd\u4f5c\u3002</p> <pre><code># kubectl ko nb status\n1b9a\nName: OVN_Northbound\nCluster ID: 32ca (32ca07fb-739b-4257-b510-12fa18e7cce8)\nServer ID: 1b9a (1b9a5d76-e69b-410c-8085-39943d0cd38c)\nAddress: tcp:[172.18.0.3]:6643\nStatus: cluster member\nRole: leader\nTerm: 44\nLeader: self\nVote: self\n\nLast Election started 1855739 ms ago, reason: timeout\nLast Election won: 1855729 ms ago\nElection timer: 5000\nLog: [147, 147]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: -&gt;4984 &lt;-4984\nDisconnections: 0\nServers:\n    4984 (4984 at tcp:[172.18.0.4]:6643) next_index=147 match_index=146 last msg 367 ms ago\n    1b9a (1b9a at tcp:[172.18.0.3]:6643) (self) next_index=140 match_index=146\nstatus: ok\n\n# kubectl ko sb status\n3722\nName: OVN_Southbound\nCluster ID: d4bd (d4bd37a4-0400-499f-b4df-b4fd389780f0)\nServer ID: 3722 (3722d5ae-2ced-4820-a6b2-8b744d11fb3e)\nAddress: tcp:[172.18.0.3]:6644\nStatus: cluster member\nRole: leader\nTerm: 33\nLeader: self\nVote: self\n\nLast Election started 1868589 ms ago, reason: timeout\nLast Election won: 1868579 ms ago\nElection timer: 5000\nLog: [142, 142]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: -&gt;6e84 &lt;-6e84\nDisconnections: 0\nServers:\n    6e84 (6e84 at tcp:[172.18.0.4]:6644) next_index=142 match_index=141 last msg 728 ms ago\n    3722 (3722 at tcp:[172.18.0.3]:6644) (self) next_index=134 match_index=141\nstatus: ok\n</code></pre>"},{"location":"ops/change-ovn-central-node/#_3","title":"\u7ed9\u8282\u70b9\u589e\u52a0\u6807\u7b7e\u5e76\u6269\u5bb9","text":"<p>\u6ce8\u610f\u9700\u5728 ovn-central \u73af\u5883\u53d8\u91cf <code>NODE_IPS</code> \u7684\u8282\u70b9\u5730\u5740\u4e2d\u589e\u52a0\u4e0a\u7ebf\u8282\u70b9\u5730\u5740\u3002</p> <pre><code>kubectl label node kube-ovn-control-plane2 kube-ovn/role=master\nkubectl scale deployment -n kube-system ovn-central --replicas=3\nkubectl set env deployment/ovn-central -n kube-system NODE_IPS=\"172.18.0.3,172.18.0.4,172.18.0.5\"\nkubectl rollout status deployment/ovn-central -n kube-system\n</code></pre>"},{"location":"ops/change-ovn-central-node/#ovn-central_6","title":"\u4fee\u6539\u5176\u4ed6\u7ec4\u4ef6\u8fde\u63a5 ovn-central \u5730\u5740","text":"<p>\u4fee\u6539 <code>ovs-ovn</code> \u5185\u8fde\u63a5\u4fe1\u606f\uff0c\u589e\u52a0\u4e0a\u7ebf\u8282\u70b9\u5730\u5740\uff1a</p> <pre><code># kubectl set env daemonset/ovs-ovn -n kube-system OVN_DB_IPS=\"172.18.0.3,172.18.0.4,172.18.0.5\"\ndaemonset.apps/ovs-ovn env updated\n# kubectl delete pod -n kube-system -lapp=ovs\npod \"ovs-ovn-4f6jc\" deleted\npod \"ovs-ovn-csn2w\" deleted\npod \"ovs-ovn-mpbmb\" deleted\n</code></pre> <p>\u4fee\u6539 <code>kube-ovn-controller</code> \u5185\u8fde\u63a5\u4fe1\u606f\uff0c\u589e\u52a0\u4e0a\u7ebf\u8282\u70b9\u5730\u5740\uff1a</p> <pre><code># kubectl set env deployment/kube-ovn-controller -n kube-system OVN_DB_IPS=\"172.18.0.3,172.18.0.4,172.18.0.5\"\ndeployment.apps/kube-ovn-controller env updated\n\n# kubectl rollout status deployment/kube-ovn-controller -n kube-system\nWaiting for deployment \"kube-ovn-controller\" rollout to finish: 1 of 3 updated replicas are available...\nWaiting for deployment \"kube-ovn-controller\" rollout to finish: 2 of 3 updated replicas are available...\ndeployment \"kube-ovn-controller\" successfully rolled out\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"ops/delete-worker-node/","title":"\u5220\u9664\u5de5\u4f5c\u8282\u70b9","text":"<p>\u5982\u679c\u53ea\u662f\u7b80\u5355\u4ece Kubernetes \u4e2d\u5220\u9664\u8282\u70b9\uff0c\u7531\u4e8e\u8282\u70b9\u4e0a <code>ovs-ovn</code> \u4e2d\u8fd0\u884c\u7684 <code>ovn-controller</code> \u8fdb\u7a0b\u4ecd\u5728\u8fd0\u884c\u4f1a\u5b9a\u671f\u8fde\u63a5 <code>ovn-central</code> \u6ce8\u518c\u76f8\u5173\u7f51\u7edc\u4fe1\u606f\uff0c \u4f1a\u5bfc\u81f4\u989d\u5916\u8d44\u6e90\u6d6a\u8d39\u5e76\u6709\u6f5c\u5728\u7684\u89c4\u5219\u51b2\u7a81\u98ce\u9669\u3002 \u56e0\u6b64\u5728\u4ece Kubernetes \u5185\u5220\u9664\u8282\u70b9\u65f6\uff0c\u8bf7\u6309\u7167\u4e0b\u9762\u7684\u6b65\u9aa4\u6765\u4fdd\u8bc1\u7f51\u7edc\u4fe1\u606f\u53ef\u4ee5\u6b63\u5e38\u88ab\u6e05\u7406\u3002</p> <p>\u8be5\u6587\u6863\u4ecb\u7ecd\u5220\u9664\u5de5\u4f5c\u8282\u70b9\u7684\u6b65\u9aa4\uff0c\u5982\u9700\u66f4\u6362 <code>ovn-central</code> \u6240\u5728\u8282\u70b9\uff0c\u8bf7\u53c2\u8003\u66f4\u6362 ovn-central \u8282\u70b9\u3002</p>"},{"location":"ops/delete-worker-node/#_2","title":"\u9a71\u9010\u8282\u70b9\u4e0a\u6240\u6709\u5bb9\u5668","text":"<pre><code> # kubectl drain kube-ovn-worker --ignore-daemonsets --force\nnode/kube-ovn-worker cordoned\n WARNING: ignoring DaemonSet-managed Pods: kube-system/kube-ovn-cni-zt74b, kube-system/kube-ovn-pinger-5rxfs, kube-system/kube-proxy-jpmnm, kube-system/ovs-ovn-v2kll\n evicting pod kube-system/coredns-64897985d-qsgpt\n evicting pod local-path-storage/local-path-provisioner-5ddd94ff66-llss6\n evicting pod kube-system/kube-ovn-controller-8459db5ff4-94lxb\n pod/kube-ovn-controller-8459db5ff4-94lxb evicted\n pod/coredns-64897985d-qsgpt evicted\n pod/local-path-provisioner-5ddd94ff66-llss6 evicted\n node/kube-ovn-worker drained\n</code></pre>"},{"location":"ops/delete-worker-node/#kubelet-docker","title":"\u505c\u6b62 kubelet \u548c docker","text":"<p>\u8be5\u6b65\u9aa4\u4f1a\u505c\u6b62 <code>ovs-ovn</code> \u5bb9\u5668\uff0c\u4ee5\u907f\u514d\u5411 <code>ovn-central</code> \u8fdb\u884c\u4fe1\u606f\u6ce8\u518c\uff0c\u767b\u5f55\u5230\u5bf9\u5e94\u8282\u70b9\u6267\u884c\u4e0b\u5217\u547d\u4ee4\uff1a</p> <pre><code>systemctl stop kubelet\nsystemctl stop docker\n</code></pre> <p>\u5982\u679c\u4f7f\u7528\u7684 CRI \u4e3a containerd\uff0c\u9700\u8981\u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u505c\u6b62 <code>ovs-ovn</code> \u5bb9\u5668\uff1a</p> <pre><code>crictl rm -f $(crictl ps | grep openvswitch | awk '{print $1}')\n</code></pre>"},{"location":"ops/delete-worker-node/#node","title":"\u6e05\u7406 Node \u4e0a\u7684\u6b8b\u7559\u6570\u636e","text":"<pre><code>rm -rf /var/run/openvswitch\nrm -rf /var/run/ovn\nrm -rf /etc/origin/openvswitch/\nrm -rf /etc/origin/ovn/\nrm -rf /etc/cni/net.d/00-kube-ovn.conflist\nrm -rf /etc/cni/net.d/01-kube-ovn.conflist\nrm -rf /var/log/openvswitch\nrm -rf /var/log/ovn\n</code></pre>"},{"location":"ops/delete-worker-node/#kubectl","title":"\u4f7f\u7528 kubectl \u5220\u9664\u8282\u70b9","text":"<pre><code>kubectl delete no kube-ovn-01\n</code></pre>"},{"location":"ops/delete-worker-node/#ovn-sb","title":"\u68c0\u67e5\u5bf9\u5e94\u8282\u70b9\u662f\u5426\u4ece ovn-sb \u4e2d\u5220\u9664","text":"<p>\u4e0b\u9762\u7684\u793a\u4f8b\u4e3a <code>kube-ovn-worker</code> \u4f9d\u7136\u672a\u88ab\u5220\u9664\uff1a</p> <pre><code># kubectl ko sbctl show\nChassis \"b0564934-5a0d-4804-a4c0-476c93596a17\"\nhostname: kube-ovn-worker\n  Encap geneve\n      ip: \"172.18.0.2\"\noptions: {csum=\"true\"}\nPort_Binding kube-ovn-pinger-5rxfs.kube-system\nChassis \"6a29de7e-d731-4eaf-bacd-2f239ee52b28\"\nhostname: kube-ovn-control-plane\n  Encap geneve\n      ip: \"172.18.0.3\"\noptions: {csum=\"true\"}\nPort_Binding coredns-64897985d-nbfln.kube-system\n  Port_Binding node-kube-ovn-control-plane\n  Port_Binding local-path-provisioner-5ddd94ff66-h4tn9.local-path-storage\n  Port_Binding kube-ovn-pinger-hf2p6.kube-system\n  Port_Binding coredns-64897985d-fhwlw.kube-system\n</code></pre>"},{"location":"ops/delete-worker-node/#chassis","title":"\u82e5\u8282\u70b9\u5bf9\u5e94\u7684 chassis \u4f9d\u7136\u5b58\u5728\uff0c\u624b\u52a8\u8fdb\u884c\u5220\u9664","text":"<p>uuid \u4e3a\u4e4b\u524d\u547d\u4ee4\u6240\u67e5\u51fa\u7684 Chassis \u5bf9\u5e94 id\uff1a</p> <pre><code># kubectl ko sbctl chassis-del b0564934-5a0d-4804-a4c0-476c93596a17\n# kubectl ko sbctl show\nChassis \"6a29de7e-d731-4eaf-bacd-2f239ee52b28\"\nhostname: kube-ovn-control-plane\n  Encap geneve\n      ip: \"172.18.0.3\"\noptions: {csum=\"true\"}\nPort_Binding coredns-64897985d-nbfln.kube-system\n  Port_Binding node-kube-ovn-control-plane\n  Port_Binding local-path-provisioner-5ddd94ff66-h4tn9.local-path-storage\n  Port_Binding kube-ovn-pinger-hf2p6.kube-system\n  Port_Binding coredns-64897985d-fhwlw.kube-system\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"ops/faq/","title":"\u5176\u4ed6\u5e38\u89c1\u95ee\u9898","text":""},{"location":"ops/faq/#arm","title":"\u9e92\u9e9f ARM \u7cfb\u7edf\u8de8\u4e3b\u673a\u5bb9\u5668\u8bbf\u95ee\u95f4\u6b47\u5931\u8d25","text":""},{"location":"ops/faq/#_2","title":"\u73b0\u8c61","text":"<p>\u9e92\u9e9f ARM \u7cfb\u7edf\u548c\u90e8\u5206\u56fd\u4ea7\u5316\u7f51\u5361 offload \u914d\u5408\u5b58\u5728\u95ee\u9898\uff0c\u4f1a\u5bfc\u81f4\u5bb9\u5668\u7f51\u7edc\u95f4\u6b47\u6545\u969c\u3002</p> <p>\u4f7f\u7528 <code>netstat</code> \u786e\u8ba4\u95ee\u9898\uff1a</p> <pre><code># netstat -us\nIcmpMsg:\n    InType0: 22\nInType3: 24\nInType8: 117852\nOutType0: 117852\nOutType3: 29\nOutType8: 22\nUdp:\n    3040636 packets received\n    0 packets to unknown port received.\n    4 packet receive errors\n    602 packets sent\n    0 receive buffer errors\n    0 send buffer errors\n    InCsumErrors: 4\nUdpLite:\nIpExt:\n    InBcastPkts: 10244\nInOctets: 4446320361\nOutOctets: 1496815600\nInBcastOctets: 3095950\nInNoECTPkts: 7683903\n</code></pre> <p>\u82e5\u5b58\u5728 <code>InCsumErrors</code>\uff0c\u4e14\u968f\u7740\u8bbf\u95ee\u5931\u8d25\u589e\u52a0\uff0c\u53ef\u786e\u8ba4\u662f\u8be5\u95ee\u9898\u3002</p>"},{"location":"ops/faq/#_3","title":"\u89e3\u51b3\u65b9\u6cd5","text":"<p>\u6839\u672c\u89e3\u51b3\u9700\u8981\u548c\u9e92\u9e9f\u4ee5\u53ca\u5bf9\u5e94\u7f51\u5361\u5382\u5546\u6c9f\u901a\uff0c\u66f4\u65b0\u7cfb\u7edf\u548c\u9a71\u52a8\u3002\u4e34\u65f6\u89e3\u51b3\u53ef\u5148\u5173\u95ed\u7269\u7406 \u7f51\u5361\u7684 <code>tx offload</code> \u4f46\u662f\u4f1a\u5bfc\u81f4 tcp \u6027\u80fd\u6709\u8f83\u660e\u663e\u4e0b\u964d\u3002</p> <pre><code>ethtool -K eth0 tx off\n</code></pre> <p>\u7ecf\u793e\u533a\u53cd\u9988\u4f7f\u7528 <code>4.19.90-25.16.v2101</code> \u5185\u6838\u540e\u53ef\u4ee5\u89e3\u51b3\u8be5\u95ee\u9898\u3002</p>"},{"location":"ops/faq/#pod-service","title":"Pod \u8bbf\u95ee Service \u4e0d\u901a","text":""},{"location":"ops/faq/#_4","title":"\u73b0\u8c61","text":"<p>Pod \u5185\u65e0\u6cd5\u8bbf\u95ee Service \u5bf9\u5e94\u7684\u670d\u52a1\uff0c<code>dmesg</code> \u663e\u793a\u5f02\u5e38\uff1a</p> <pre><code>netlink: Unknown conntrack attr (type=6, max=5)\nopenvswitch: netlink: Flow actions may not be safe on all matching packets.\n</code></pre> <p>\u8be5\u65e5\u5fd7\u8bf4\u660e\u5185\u6838\u5185 OVS \u7248\u672c\u8fc7\u4f4e\u4e0d\u652f\u6301\u5bf9\u5e94 NAT \u64cd\u4f5c\u3002</p>"},{"location":"ops/faq/#_5","title":"\u89e3\u51b3\u65b9\u6cd5","text":"<ol> <li>\u5347\u7ea7\u5185\u6838\u6a21\u5757\u6216\u624b\u52a8\u7f16\u8bd1 OVS \u5185\u6838\u6a21\u5757\u3002</li> <li>\u82e5\u53ea\u4f7f\u7528 Overlay \u7f51\u7edc\u53ef\u4ee5\u66f4\u6539 <code>kube-ovn-controller</code> \u542f\u52a8\u53c2\u6570\u8bbe\u7f6e <code>--enable-lb=false</code> \u5173\u95ed OVN LB \u4f7f\u7528 kube-proxy \u8fdb\u884c Service \u8f6c\u53d1\u3002</li> </ol> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"ops/from-calico/","title":"\u5378\u8f7d Calico \u5b89\u88c5 Kube-OVN","text":"<p>\u82e5 Kubernetes \u96c6\u7fa4\u5df2\u5b89\u88c5 Calico \u9700\u8981\u53d8\u66f4\u4e3a Kube-OVN \u53ef\u4ee5\u53c2\u8003\u672c\u6587\u6863\u3002</p> <p>\u672c\u6587\u4ee5 Calico v3.24.1 \u4e3a\u4f8b\uff0c\u5176\u5b83 Calico \u7248\u672c\u9700\u8981\u6839\u636e\u5b9e\u9645\u60c5\u51b5\u8fdb\u884c\u8c03\u6574\u3002</p>"},{"location":"ops/from-calico/#_1","title":"\u51c6\u5907\u5de5\u4f5c","text":"<p>\u4e3a\u4e86\u4fdd\u8bc1\u5207\u6362 CNI \u8fc7\u7a0b\u4e2d\u96c6\u7fa4\u7f51\u7edc\u4fdd\u6301\u7545\u901a\uff0cCalico ippool \u9700\u8981\u5f00\u542f nat outgoing\uff0c\u6216\u5728\u6240\u6709\u8282\u70b9\u4e0a\u5173\u95ed rp_filter\uff1a</p> <pre><code>sysctl net.ipv4.conf.all.rp_filter=0\nsysctl net.ipv4.conf.default.rp_filter=0\n# IPIP \u6a21\u5f0f\nsysctl net.ipv4.conf.tunl0.rp_filter=0\n# VXLAN \u6a21\u5f0f\nsysctl net.ipv4.conf.vxlan/calico.rp_filter=0\n# \u8def\u7531\u6a21\u5f0f\uff0ceth0 \u9700\u8981\u4fee\u6539\u4e3a\u5b9e\u9645\u4f7f\u7528\u7684\u7f51\u5361\nsysctl net.ipv4.conf.eth0.rp_filter=0\n</code></pre>"},{"location":"ops/from-calico/#kube-ovn","title":"\u90e8\u7f72 Kube-OVN","text":""},{"location":"ops/from-calico/#_2","title":"\u4e0b\u8f7d\u5b89\u88c5\u811a\u672c","text":"<pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/images/install.sh\n</code></pre>"},{"location":"ops/from-calico/#_3","title":"\u4fee\u6539\u5b89\u88c5\u811a\u672c","text":"<p>\u5c06\u5b89\u88c5\u811a\u672c\u4e2d\u91cd\u5efa Pod \u7684\u90e8\u5206\u5220\u9664\uff1a</p> <pre><code>echo \"[Step 4/6] Delete pod that not in host network mode\"\nfor ns in $(kubectl get ns --no-headers -o custom-columns=NAME:.metadata.name); do\nfor pod in $(kubectl get pod --no-headers -n \"$ns\" --field-selector spec.restartPolicy=Always -o custom-columns=NAME:.metadata.name,HOST:spec.hostNetwork | awk '{if ($2!=\"true\") print $1}'); do\nkubectl delete pod \"$pod\" -n \"$ns\" --ignore-not-found\n  done\ndone\n</code></pre> <p>\u6309\u9700\u4fee\u6539\u4ee5\u4e0b\u914d\u7f6e\uff1a</p> <pre><code>REGISTRY=\"kubeovn\"                     # \u955c\u50cf\u4ed3\u5e93\u5730\u5740\nVERSION=\"v1.13.0\"                      # \u955c\u50cf\u7248\u672c/Tag\nPOD_CIDR=\"10.16.0.0/16\"                # \u9ed8\u8ba4\u5b50\u7f51 CIDR \u4e0d\u8981\u548c SVC/NODE/JOIN CIDR \u91cd\u53e0\nSVC_CIDR=\"10.96.0.0/12\"                # \u9700\u8981\u548c apiserver \u7684 service-cluster-ip-range \u4fdd\u6301\u4e00\u81f4\nJOIN_CIDR=\"100.64.0.0/16\"              # Pod \u548c\u4e3b\u673a\u901a\u4fe1\u7f51\u7edc CIDR\uff0c\u4e0d\u8981\u548c SVC/NODE/POD CIDR \u91cd\u53e0 \nLABEL=\"node-role.kubernetes.io/master\" # \u90e8\u7f72 OVN DB \u8282\u70b9\u7684\u6807\u7b7e\nIFACE=\"\"                               # \u5bb9\u5668\u7f51\u7edc\u6240\u4f7f\u7528\u7684\u7684\u5bbf\u4e3b\u673a\u7f51\u5361\u540d\uff0c\u5982\u679c\u4e3a\u7a7a\u5219\u4f7f\u7528 Kubernetes \u4e2d\u7684 Node IP \u6240\u5728\u7f51\u5361\nTUNNEL_TYPE=\"geneve\"                   # \u96a7\u9053\u5c01\u88c5\u534f\u8bae\uff0c\u53ef\u9009 geneve, vxlan \u6216 stt\uff0cstt \u9700\u8981\u5355\u72ec\u7f16\u8bd1 ovs \u5185\u6838\u6a21\u5757\n</code></pre> <p>\u6ce8\u610f\uff1aPOD_CIDR \u53ca JOIN_CIDR \u4e0d\u53ef\u4e0e Calico ippool \u7684 CIDR \u51b2\u7a81\uff0c\u4e14 POD_CIDR \u9700\u8981\u5305\u542b\u8db3\u591f\u591a\u7684 IP \u6765\u5bb9\u7eb3\u96c6\u7fa4\u4e2d\u5df2\u6709\u7684 Pod\u3002</p>"},{"location":"ops/from-calico/#_4","title":"\u6267\u884c\u5b89\u88c5\u811a\u672c","text":"<pre><code>bash install.sh\n</code></pre>"},{"location":"ops/from-calico/#_5","title":"\u9010\u4e2a\u8282\u70b9\u8fc1\u79fb","text":"<p>\u6309\u7167\u4ee5\u4e0b\u65b9\u6cd5\u4e3a\u6bcf\u4e2a\u8282\u70b9\u9010\u4e2a\u8fdb\u884c\u8fc1\u79fb\u3002 \u6ce8\u610f\uff1a\u547d\u4ee4\u4e2d\u7684 \\&lt;NODE&gt; \u9700\u8981\u66ff\u6362\u4e3a\u8282\u70b9\u540d\u79f0\u3002</p>"},{"location":"ops/from-calico/#_6","title":"\u9a71\u9010\u8282\u70b9","text":"<pre><code>kubectl drain --ignore-daemonsets &lt;NODE&gt;\n</code></pre> <p>\u82e5\u6b64\u547d\u4ee4\u4e00\u76f4\u7b49\u5f85 Pod \u88ab\u9a71\u9010\uff0c\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u5f3a\u5236\u5220\u9664\u88ab\u9a71\u9010\u7684 Pod\uff1a</p> <pre><code>kubectl get pod -A --field-selector=spec.nodeName=&lt;NODE&gt; --no-headers | \\\nawk '$4==\"Terminating\" {print $1\" \"$2}' | \\\nwhile read s; do kubectl delete pod --force -n $s; done\n</code></pre>"},{"location":"ops/from-calico/#_7","title":"\u91cd\u542f\u8282\u70b9","text":"<p>\u5728\u8282\u70b9\u4e2d\u6267\u884c\uff1a</p> <pre><code>shutdown -r 0\n</code></pre>"},{"location":"ops/from-calico/#_8","title":"\u6062\u590d\u8282\u70b9","text":"<pre><code>kubectl uncordon &lt;NODE&gt;\n</code></pre>"},{"location":"ops/from-calico/#calico","title":"\u5378\u8f7d Calico","text":""},{"location":"ops/from-calico/#k8s","title":"\u5220\u9664 k8s \u8d44\u6e90","text":"<pre><code>kubectl -n kube-system delete deploy calico-kube-controllers\nkubectl -n kube-system delete ds calico-node\nkubectl -n kube-system delete cm calico-config\n# \u5220\u9664 CRD \u53ca\u76f8\u5173\u8d44\u6e90\nkubectl get crd -o jsonpath='{range .items[*]}{.metadata.name}{\"\\n\"}{end}' | while read crd; do\nif ! echo $crd | grep '.crd.projectcalico.org$' &gt;/dev/null; then\ncontinue\nfi\n\nfor name in $(kubectl get $crd -o jsonpath='{.items[*].metadata.name}'); do\nkubectl delete $crd $name\ndone\nkubectl delete crd $crd\ndone\n# \u5176\u5b83\u8d44\u6e90\nkubectl delete --ignore-not-found clusterrolebinding calico-node calico-kube-controllers\nkubectl delete --ignore-not-found clusterrole calico-node calico-kube-controllers\nkubectl delete --ignore-not-found sa -n kube-system calico-kube-controllers calico-node\nkubectl delete --ignore-not-found pdb -n kube-system calico-kube-controllers\n</code></pre>"},{"location":"ops/from-calico/#_9","title":"\u6e05\u7406\u8282\u70b9\u6587\u4ef6","text":"<p>\u5728\u6bcf\u4e2a\u8282\u70b9\u4e2d\u6267\u884c\uff1a</p> <pre><code>rm -f /etc/cni/net.d/10-calico.conflist /etc/cni/net.d/calico-kubeconfig\nrm -f /opt/cni/bin/calico /opt/cni/bin/calico-ipam\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"ops/kubectl-ko/","title":"kubectl \u63d2\u4ef6\u4f7f\u7528","text":"<p>\u4e3a\u4e86\u65b9\u4fbf\u65e5\u5e38\u7684\u8fd0\u7ef4\u64cd\u4f5c\uff0cKube-OVN \u63d0\u4f9b\u4e86 kubectl \u63d2\u4ef6\u5de5\u5177\uff0c\u7f51\u7edc\u7ba1\u7406\u5458 \u53ef\u4ee5\u901a\u8fc7\u8be5\u547d\u4ee4\u8fdb\u884c\u65e5\u5e38\u64cd\u4f5c\uff0c\u4f8b\u5982\uff1a\u67e5\u770b OVN \u6570\u636e\u5e93\u4fe1\u606f\u548c\u72b6\u6001\uff0cOVN \u6570\u636e\u5e93 \u5907\u4efd\u548c\u6062\u590d\uff0cOVS \u76f8\u5173\u4fe1\u606f\u67e5\u770b\uff0ctcpdump \u7279\u5b9a\u5bb9\u5668\uff0c\u7279\u5b9a\u94fe\u8def\u903b\u8f91\u62d3\u6251\u5c55\u793a\uff0c \u7f51\u7edc\u95ee\u9898\u8bca\u65ad\u548c\u6027\u80fd\u4f18\u5316\u3002</p>"},{"location":"ops/kubectl-ko/#_1","title":"\u63d2\u4ef6\u5b89\u88c5","text":"<p>Kube-OVN \u5b89\u88c5\u65f6\u9ed8\u8ba4\u4f1a\u90e8\u7f72\u63d2\u4ef6\u5230\u6bcf\u4e2a\u8282\u70b9\uff0c\u82e5\u6267\u884c kubectl \u7684\u673a\u5668\u4e0d\u5728\u96c6\u7fa4\u5185\uff0c \u6216\u9700\u8981\u91cd\u88c5\u63d2\u4ef6\uff0c\u53ef\u53c2\u8003\u4e0b\u9762\u7684\u6b65\u9aa4\uff1a</p> <p>\u4e0b\u8f7d <code>kubectl-ko</code> \u6587\u4ef6\uff1a</p> <pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/images/kubectl-ko\n</code></pre> <p>\u5c06\u8be5\u6587\u4ef6\u79fb\u52a8\u81f3 <code>$PATH</code> \u76ee\u5f55\u4e0b\uff1a</p> <pre><code>mv kubectl-ko /usr/local/bin/kubectl-ko\n</code></pre> <p>\u589e\u52a0\u53ef\u6267\u884c\u6743\u9650\uff1a</p> <pre><code>chmod +x /usr/local/bin/kubectl-ko\n</code></pre> <p>\u68c0\u67e5\u63d2\u4ef6\u662f\u5426\u53ef\u4ee5\u6b63\u5e38\u4f7f\u7528\uff1a</p> <pre><code># kubectl plugin list\nThe following compatible plugins are available:\n\n/usr/local/bin/kubectl-ko\n</code></pre>"},{"location":"ops/kubectl-ko/#_2","title":"\u63d2\u4ef6\u4f7f\u7528","text":"<p>\u8fd0\u884c <code>kubectl ko</code> \u4f1a\u5c55\u793a\u8be5\u63d2\u4ef6\u6240\u6709\u53ef\u7528\u7684\u547d\u4ee4\u548c\u7528\u6cd5\u63cf\u8ff0\uff0c\u5982\u4e0b\u6240\u793a\uff1a</p> <pre><code># kubectl ko\nkubectl ko {subcommand} [option...]\nAvailable Subcommands:\n  [nb|sb] [status|kick|backup|dbstatus|restore]     ovn-db operations show cluster status, kick stale server, backup database, get db consistency status or restore ovn nb db when met 'inconsistent data' error\n  nbctl [ovn-nbctl options ...]    invoke ovn-nbctl\n  sbctl [ovn-sbctl options ...]    invoke ovn-sbctl\n  vsctl {nodeName} [ovs-vsctl options ...]   invoke ovs-vsctl on the specified node\n  ofctl {nodeName} [ovs-ofctl options ...]   invoke ovs-ofctl on the specified node\n  dpctl {nodeName} [ovs-dpctl options ...]   invoke ovs-dpctl on the specified node\n  appctl {nodeName} [ovs-appctl options ...]   invoke ovs-appctl on the specified node\n  tcpdump {namespace/podname} [tcpdump options ...]     capture pod traffic\n  trace {namespace/podname} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp/udp port]    trace ICMP/TCP/UDP\n  trace {namespace/podname} {target ip address} [target mac address] arp {request|reply}                     trace ARP request/reply\n  trace {node//nodename} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp/udp port]       trace ICMP/TCP/UDP\n  trace {node//nodename} {target ip address} [target mac address] arp {request|reply}                        trace ARP request/reply\n  echo \"  diagnose {all|node|subnet|IPPorts} [nodename|subnetName|{proto1}-{IP1}-{Port1},{proto2}-{IP2}-{Port2}]    diagnose connectivity of all nodes or a specific node or specify subnet's ds pod or IPPorts like 'tcp-172.18.0.2-53,udp-172.18.0.3-53'\"\ntuning {install-fastpath|local-install-fastpath|remove-fastpath|install-stt|local-install-stt|remove-stt} {centos7|centos8}} [kernel-devel-version]  deploy  kernel optimisation components to the system\n  reload    restart all kube-ovn components\n  log {kube-ovn|ovn|ovs|linux|all}    save log to ./kubectl-ko-log/\n  perf [image] performance test default image is kubeovn/test:v1.12.0  </code></pre> <p>\u4e0b\u9762\u5c06\u4ecb\u7ecd\u6bcf\u4e2a\u547d\u4ee4\u7684\u5177\u4f53\u529f\u80fd\u548c\u4f7f\u7528\u3002</p>"},{"location":"ops/kubectl-ko/#nb-sb-status-kick-backup-dbstatus-restore","title":"[nb | sb] [status | kick | backup | dbstatus | restore]","text":"<p>\u8be5\u5b50\u547d\u4ee4\u4e3b\u8981\u5bf9 OVN \u5317\u5411\u6216\u5357\u5411\u6570\u636e\u5e93\u8fdb\u884c\u64cd\u4f5c\uff0c\u5305\u62ec\u6570\u636e\u5e93\u96c6\u7fa4\u72b6\u6001\u67e5\u770b\uff0c\u6570\u636e\u5e93\u8282\u70b9\u4e0b\u7ebf\uff0c \u6570\u636e\u5e93\u5907\u4efd\uff0c\u6570\u636e\u5e93\u5b58\u50a8\u72b6\u6001\u67e5\u770b\u548c\u6570\u636e\u5e93\u4fee\u590d\u3002</p>"},{"location":"ops/kubectl-ko/#_3","title":"\u6570\u636e\u5e93\u96c6\u7fa4\u72b6\u6001\u67e5\u770b","text":"<p>\u8be5\u547d\u4ee4\u4f1a\u5728\u5bf9\u5e94 OVN \u6570\u636e\u5e93\u7684 leader \u8282\u70b9\u6267\u884c <code>ovs-appctl cluster/status</code> \u5c55\u793a\u96c6\u7fa4\u72b6\u6001:</p> <pre><code># kubectl ko nb status\n306b\nName: OVN_Northbound\nCluster ID: 9a87 (9a872522-3e7d-47ca-83a3-d74333e1a7ca)\nServer ID: 306b (306b256b-b5e1-4eb0-be91-4ca96adf6bad)\nAddress: tcp:[172.18.0.2]:6643\nStatus: cluster member\nRole: leader\nTerm: 1\nLeader: self\nVote: self\n\nLast Election started 280309 ms ago, reason: timeout\nLast Election won: 280309 ms ago\nElection timer: 5000\nLog: [139, 139]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: &lt;-8723 -&gt;8723 &lt;-85d6 -&gt;85d6\nDisconnections: 0\nServers:\n    85d6 (85d6 at tcp:[172.18.0.4]:6643) next_index=139 match_index=138 last msg 763 ms ago\n    8723 (8723 at tcp:[172.18.0.3]:6643) next_index=139 match_index=138 last msg 763 ms ago\n    306b (306b at tcp:[172.18.0.2]:6643) (self) next_index=2 match_index=138\nstatus: ok\n</code></pre> <p>\u82e5 <code>Server</code> \u4e0b\u7684 <code>match_index</code> \u51fa\u73b0\u8f83\u5927\u5dee\u522b\uff0c\u4e14 <code>last msg</code> \u65f6\u95f4\u8f83\u957f\u5219\u5bf9\u5e94 Server \u53ef\u80fd\u957f\u65f6\u95f4\u6ca1\u6709\u54cd\u5e94\uff0c \u9700\u8981\u8fdb\u4e00\u6b65\u67e5\u770b\u3002</p>"},{"location":"ops/kubectl-ko/#_4","title":"\u6570\u636e\u5e93\u8282\u70b9\u4e0b\u7ebf","text":"<p>\u8be5\u547d\u4ee4\u4f1a\u5c06\u67d0\u4e2a\u8282\u70b9\u4ece OVN \u6570\u636e\u5e93\u4e2d\u79fb\u9664\uff0c\u5728\u8282\u70b9\u4e0b\u7ebf\u6216\u66f4\u6362\u8282\u70b9\u65f6\u9700\u8981\u7528\u5230\u3002 \u4e0b\u9762\u5c06\u4ee5\u4e0a\u4e00\u6761\u547d\u4ee4\u6240\u67e5\u770b\u5230\u7684\u96c6\u7fa4\u72b6\u6001\u4e3a\u4f8b\uff0c\u4e0b\u7ebf <code>172.18.0.3</code> \u8282\u70b9:</p> <pre><code># kubectl ko nb kick 8723\nstarted removal\n</code></pre> <p>\u518d\u6b21\u67e5\u770b\u6570\u636e\u5e93\u96c6\u7fa4\u72b6\u6001\u786e\u8ba4\u8282\u70b9\u5df2\u79fb\u9664\uff1a</p> <pre><code># kubectl ko nb status\n306b\nName: OVN_Northbound\nCluster ID: 9a87 (9a872522-3e7d-47ca-83a3-d74333e1a7ca)\nServer ID: 306b (306b256b-b5e1-4eb0-be91-4ca96adf6bad)\nAddress: tcp:[172.18.0.2]:6643\nStatus: cluster member\nRole: leader\nTerm: 1\nLeader: self\nVote: self\n\nLast Election started 324356 ms ago, reason: timeout\nLast Election won: 324356 ms ago\nElection timer: 5000\nLog: [140, 140]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: &lt;-85d6 -&gt;85d6\nDisconnections: 2\nServers:\n    85d6 (85d6 at tcp:[172.18.0.4]:6643) next_index=140 match_index=139 last msg 848 ms ago\n    306b (306b at tcp:[172.18.0.2]:6643) (self) next_index=2 match_index=139\nstatus: ok\n</code></pre>"},{"location":"ops/kubectl-ko/#_5","title":"\u6570\u636e\u5e93\u5907\u4efd","text":"<p>\u8be5\u5b50\u547d\u4ee4\u4f1a\u5907\u4efd\u5f53\u524d OVN \u6570\u636e\u5e93\u81f3\u672c\u5730\uff0c\u53ef\u7528\u4e8e\u707e\u5907\u548c\u6062\u590d\uff1a</p> <pre><code># kubectl ko nb backup\ntar: Removing leading `/' from member names\nbackup ovn-nb db to /root/ovnnb_db.060223191654183154.backup\n</code></pre>"},{"location":"ops/kubectl-ko/#_6","title":"\u6570\u636e\u5e93\u5b58\u50a8\u72b6\u6001\u67e5\u770b","text":"<p>\u8be5\u547d\u4ee4\u7528\u6765\u67e5\u770b\u6570\u636e\u5e93\u6587\u4ef6\u662f\u5426\u5b58\u5728\u635f\u574f\uff1a</p> <pre><code># kubectl ko nb dbstatus\nstatus: ok\n</code></pre> <p>\u82e5\u5f02\u5e38\u5219\u663e\u793a <code>inconsistent data</code> \u9700\u8981\u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u4fee\u590d\u3002</p>"},{"location":"ops/kubectl-ko/#_7","title":"\u6570\u636e\u5e93\u4fee\u590d","text":"<p>\u82e5\u6570\u636e\u5e93\u72b6\u6001\u8fdb\u5165 <code>inconsistent data</code> \u53ef\u4f7f\u7528\u8be5\u547d\u4ee4\u8fdb\u884c\u4fee\u590d\uff1a</p> <pre><code># kubectl ko nb restore\ndeployment.apps/ovn-central scaled\novn-central original replicas is 3\nfirst nodeIP is 172.18.0.5\novs-ovn pod on node 172.18.0.5 is ovs-ovn-8jxv9\novs-ovn pod on node 172.18.0.3 is ovs-ovn-sjzb6\novs-ovn pod on node 172.18.0.4 is ovs-ovn-t87zk\nbackup nb db file\nrestore nb db file, operate in pod ovs-ovn-8jxv9\ndeployment.apps/ovn-central scaled\nfinish restore nb db file and ovn-central replicas\nrecreate ovs-ovn pods\npod \"ovs-ovn-8jxv9\" deleted\npod \"ovs-ovn-sjzb6\" deleted\npod \"ovs-ovn-t87zk\" deleted\n</code></pre>"},{"location":"ops/kubectl-ko/#nbctl-sbctl-options","title":"[nbctl | sbctl] [options ...]","text":"<p>\u8be5\u5b50\u547d\u4ee4\u4f1a\u76f4\u63a5\u8fdb\u5165 OVN \u5317\u5411\u6570\u636e\u5e93\u6216\u5357\u5411\u6570\u636e\u5e93 \u7684 leader \u8282\u70b9\u5206\u522b\u6267\u884c <code>ovn-nbctl</code> \u548c <code>ovn-sbctl</code> \u547d\u4ee4\u3002 \u66f4\u591a\u8be5\u547d\u4ee4\u7684\u8be6\u7ec6\u7528\u6cd5\u8bf7\u67e5\u8be2\u4e0a\u6e38 OVN \u7684\u5b98\u65b9\u6587\u6863 ovn-nbctl(8) \u548c ovn-sbctl(8)\u3002</p> <pre><code># kubectl ko nbctl show\nswitch c7cd17e8-ceee-4a91-9bb3-e5a313fe1ece (snat)\nport snat-ovn-cluster\n        type: router\n        router-port: ovn-cluster-snat\nswitch 20e0c6d0-023a-4756-aec5-200e0c60f95d (join)\nport node-liumengxin-ovn3-192.168.137.178\n        addresses: [\"00:00:00:64:FF:A8 100.64.0.4\"]\nport node-liumengxin-ovn1-192.168.137.176\n        addresses: [\"00:00:00:AF:98:62 100.64.0.2\"]\nport node-liumengxin-ovn2-192.168.137.177\n        addresses: [\"00:00:00:D9:58:B8 100.64.0.3\"]\nport join-ovn-cluster\n        type: router\n        router-port: ovn-cluster-join\nswitch 0191705c-f827-427b-9de3-3c3b7d971ba5 (central)\nport central-ovn-cluster\n        type: router\n        router-port: ovn-cluster-central\nswitch 2a45ff05-388d-4f85-9daf-e6fccd5833dc (ovn-default)\nport alertmanager-main-0.monitoring\n        addresses: [\"00:00:00:6C:DF:A3 10.16.0.19\"]\nport kube-state-metrics-5d6885d89-4nf8h.monitoring\n        addresses: [\"00:00:00:6F:02:1C 10.16.0.15\"]\nport fake-kubelet-67c55dfd89-pv86k.kube-system\n        addresses: [\"00:00:00:5C:12:E8 10.16.19.177\"]\nport ovn-default-ovn-cluster\n        type: router\n        router-port: ovn-cluster-ovn-default\nrouter 212f73dd-d63d-4d72-864b-a537e9afbee1 (ovn-cluster)\nport ovn-cluster-snat\n        mac: \"00:00:00:7A:82:8F\"\nnetworks: [\"172.22.0.1/16\"]\nport ovn-cluster-join\n        mac: \"00:00:00:F8:18:5A\"\nnetworks: [\"100.64.0.1/16\"]\nport ovn-cluster-central\n        mac: \"00:00:00:4D:8C:F5\"\nnetworks: [\"192.101.0.1/16\"]\nport ovn-cluster-ovn-default\n        mac: \"00:00:00:A3:F8:18\"\nnetworks: [\"10.16.0.1/16\"]\n</code></pre>"},{"location":"ops/kubectl-ko/#vsctl-nodename-options","title":"vsctl {nodeName} [options ...]","text":"<p>\u8be5\u547d\u4ee4\u4f1a\u8fdb\u5165\u5bf9\u5e94 <code>nodeName</code> \u4e0a\u7684 <code>ovs-ovn</code> \u5bb9\u5668\uff0c\u5e76\u6267\u884c\u76f8\u5e94\u7684 <code>ovs-vsctl</code> \u547d\u4ee4\uff0c\u67e5\u8be2\u5e76\u914d\u7f6e <code>vswitchd</code>\u3002 \u66f4\u591a\u8be5\u547d\u4ee4\u7684\u8be6\u7ec6\u7528\u6cd5\u8bf7\u67e5\u8be2\u4e0a\u6e38 OVS \u7684\u5b98\u65b9\u6587\u6863 ovs-vsctl(8)\u3002</p> <pre><code># kubectl ko vsctl kube-ovn-01 show\n0d4c4675-c9cc-440a-8c1a-878e17f81b88\n    Bridge br-int\n        fail_mode: secure\n        datapath_type: system\n        Port a2c1a8a8b83a_h\n            Interface a2c1a8a8b83a_h\n        Port \"4fa5c4cbb1a5_h\"\nInterface \"4fa5c4cbb1a5_h\"\nPort ovn-eef07d-0\n            Interface ovn-eef07d-0\n                type: stt\n                options: {csum=\"true\", key=flow, remote_ip=\"192.168.137.178\"}\nPort ovn0\n            Interface ovn0\n                type: internal\n        Port mirror0\n            Interface mirror0\n                type: internal\n        Port ovn-efa253-0\n            Interface ovn-efa253-0\n                type: stt\n                options: {csum=\"true\", key=flow, remote_ip=\"192.168.137.177\"}\nPort br-int\n            Interface br-int\n                type: internal\n    ovs_version: \"2.17.2\"\n</code></pre>"},{"location":"ops/kubectl-ko/#ofctl-nodename-options","title":"ofctl {nodeName} [options ...]","text":"<p>\u8be5\u547d\u4ee4\u4f1a\u8fdb\u5165\u5bf9\u5e94 <code>nodeName</code> \u4e0a\u7684 <code>ovs-ovn</code> \u5bb9\u5668\uff0c\u5e76\u6267\u884c\u76f8\u5e94\u7684 <code>ovs-ofctl</code> \u547d\u4ee4\uff0c\u67e5\u8be2\u6216\u7ba1\u7406 OpenFlow\u3002 \u66f4\u591a\u8be5\u547d\u4ee4\u7684\u8be6\u7ec6\u7528\u6cd5\u8bf7\u67e5\u8be2\u4e0a\u6e38 OVS \u7684\u5b98\u65b9\u6587\u6863 ovs-ofctl(8)\u3002</p> <pre><code># kubectl ko ofctl kube-ovn-01 dump-flows br-int\nNXST_FLOW reply (xid=0x4): flags=[more]\ncookie=0xcf3429e6, duration=671791.432s, table=0, n_packets=0, n_bytes=0, idle_age=65534, hard_age=65534, priority=100,in_port=2 actions=load:0x4-&gt;NXM_NX_REG13[],load:0x9-&gt;NXM_NX_REG11[],load:0xb-&gt;NXM_NX_REG12[],load:0x4-&gt;OXM_OF_METADATA[],load:0x1-&gt;NXM_NX_REG14[],resubmit(,8)\ncookie=0xc91413c6, duration=671791.431s, table=0, n_packets=907489, n_bytes=99978275, idle_age=0, hard_age=65534, priority=100,in_port=7 actions=load:0x1-&gt;NXM_NX_REG13[],load:0x9-&gt;NXM_NX_REG11[],load:0xb-&gt;NXM_NX_REG12[],load:0x4-&gt;OXM_OF_METADATA[],load:0x4-&gt;NXM_NX_REG14[],resubmit(,8)\ncookie=0xf180459, duration=671791.431s, table=0, n_packets=17348582, n_bytes=2667811214, idle_age=0, hard_age=65534, priority=100,in_port=6317 actions=load:0xa-&gt;NXM_NX_REG13[],load:0x9-&gt;NXM_NX_REG11[],load:0xb-&gt;NXM_NX_REG12[],load:0x4-&gt;OXM_OF_METADATA[],load:0x9-&gt;NXM_NX_REG14[],resubmit(,8)\ncookie=0x7806dd90, duration=671791.431s, table=0, n_packets=3235428, n_bytes=833821312, idle_age=0, hard_age=65534, priority=100,in_port=1 actions=load:0xd-&gt;NXM_NX_REG13[],load:0x9-&gt;NXM_NX_REG11[],load:0xb-&gt;NXM_NX_REG12[],load:0x4-&gt;OXM_OF_METADATA[],load:0x3-&gt;NXM_NX_REG14[],resubmit(,8)\n...\n</code></pre>"},{"location":"ops/kubectl-ko/#dpctl-nodename-options","title":"dpctl {nodeName} [options ...]","text":"<p>\u8be5\u547d\u4ee4\u4f1a\u8fdb\u5165\u5bf9\u5e94 <code>nodeName</code> \u4e0a\u7684 <code>ovs-ovn</code> \u5bb9\u5668\uff0c\u5e76\u6267\u884c\u76f8\u5e94\u7684 <code>ovs-dpctl</code> \u547d\u4ee4\uff0c\u67e5\u8be2\u6216\u7ba1\u7406 OVS datapath\u3002 \u66f4\u591a\u8be5\u547d\u4ee4\u7684\u8be6\u7ec6\u7528\u6cd5\u8bf7\u67e5\u8be2\u4e0a\u6e38 OVS \u7684\u5b98\u65b9\u6587\u6863 ovs-dpctl(8)\u3002</p> <pre><code># kubectl ko dpctl kube-ovn-01 show\nsystem@ovs-system:\n  lookups: hit:350805055 missed:21983648 lost:73\n  flows: 105\nmasks: hit:1970748791 total:22 hit/pkt:5.29\n  port 0: ovs-system (internal)\nport 1: ovn0 (internal)\nport 2: mirror0 (internal)\nport 3: br-int (internal)\nport 4: stt_sys_7471 (stt: packet_type=ptap)\nport 5: eeb4d9e51b5d_h\n  port 6: a2c1a8a8b83a_h\n  port 7: 4fa5c4cbb1a5_h\n</code></pre>"},{"location":"ops/kubectl-ko/#appctl-nodename-options","title":"appctl {nodeName} [options ...]","text":"<p>\u8be5\u547d\u4ee4\u4f1a\u8fdb\u5165\u5bf9\u5e94 <code>nodeName</code> \u4e0a\u7684 <code>ovs-ovn</code> \u5bb9\u5668\uff0c\u5e76\u6267\u884c\u76f8\u5e94\u7684 <code>ovs-appctl</code> \u547d\u4ee4\uff0c\u6765\u64cd\u4f5c\u76f8\u5173 daemon \u8fdb\u7a0b\u3002 \u66f4\u591a\u8be5\u547d\u4ee4\u7684\u8be6\u7ec6\u7528\u6cd5\u8bf7\u67e5\u8be2\u4e0a\u6e38 OVS \u7684\u5b98\u65b9\u6587\u6863 ovs-appctl(8)\u3002</p> <pre><code># kubectl ko appctl kube-ovn-01 vlog/list\nconsole    syslog    file\n                 -------    ------    ------\nbacktrace          OFF        ERR       INFO\nbfd                OFF        ERR       INFO\nbond               OFF        ERR       INFO\nbridge             OFF        ERR       INFO\nbundle             OFF        ERR       INFO\nbundles            OFF        ERR       INFO\n...\n</code></pre>"},{"location":"ops/kubectl-ko/#tcpdump-namespacepodname-tcpdump-options","title":"tcpdump {namespace/podname} [tcpdump options ...]","text":"<p>\u8be5\u547d\u4ee4\u4f1a\u8fdb\u5165 <code>namespace/podname</code> \u6240\u5728\u673a\u5668\u7684 <code>kube-ovn-cni</code> \u5bb9\u5668\uff0c\u5e76\u6267\u884c <code>tcpdump</code> \u6293\u53d6\u5bf9\u5e94\u5bb9\u5668 veth \u7f51\u5361 \u7aef\u7684\u6d41\u91cf\uff0c\u53ef\u4ee5\u65b9\u4fbf\u6392\u67e5\u7f51\u7edc\u76f8\u5173\u95ee\u9898\uff0c\u5982\u4e0b\u6240\u793a\uff1a</p> <pre><code># kubectl ko tcpdump default/ds1-l6n7p icmp\n+ kubectl exec -it kube-ovn-cni-wlg4s -n kube-ovn -- tcpdump -nn -i d7176fe7b4e0_h icmp\ntcpdump: verbose output suppressed, use -v or -vv for full protocol decode\nlistening on d7176fe7b4e0_h, link-type EN10MB (Ethernet), capture size 262144 bytes\n06:52:36.619688 IP 100.64.0.3 &gt; 10.16.0.4: ICMP echo request, id 2, seq 1, length 64\n06:52:36.619746 IP 10.16.0.4 &gt; 100.64.0.3: ICMP echo reply, id 2, seq 1, length 64\n06:52:37.619588 IP 100.64.0.3 &gt; 10.16.0.4: ICMP echo request, id 2, seq 2, length 64\n06:52:37.619630 IP 10.16.0.4 &gt; 100.64.0.3: ICMP echo reply, id 2, seq 2, length 64\n06:52:38.619933 IP 100.64.0.3 &gt; 10.16.0.4: ICMP echo request, id 2, seq 3, length 64\n06:52:38.619973 IP 10.16.0.4 &gt; 100.64.0.3: ICMP echo reply, id 2, seq 3, length 64\n</code></pre>"},{"location":"ops/kubectl-ko/#trace-arguments","title":"trace [arguments ...]","text":"<p>\u8be5\u547d\u4ee4\u5c06\u4f1a\u6253\u5370 Pod \u6216\u8282\u70b9\u901a\u8fc7\u7279\u5b9a\u534f\u8bae\u8bbf\u95ee\u67d0\u5730\u5740\u65f6\u5bf9\u5e94\u7684 OVN \u903b\u8f91\u6d41\u8868\u548c\u6700\u7ec8\u7684 Openflow \u6d41\u8868\uff0c \u65b9\u4fbf\u5f00\u53d1\u6216\u8fd0\u7ef4\u65f6\u5b9a\u4f4d\u6d41\u8868\u76f8\u5173\u95ee\u9898\u3002</p> <p>\u652f\u6301\u7684\u547d\u4ee4\uff1a</p> <pre><code>kubectl ko trace {namespace/podname} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp/udp port]\nkubectl ko trace {namespace/podname} {target ip address} [target mac address] arp {request|reply}\nkubectl ko trace {node//nodename} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp/udp port]\nkubectl ko trace {node//nodename} {target ip address} [target mac address] arp {request|reply}\n</code></pre> <p>\u793a\u4f8b\uff1a</p> <pre><code># kubectl ko trace default/ds1-l6n7p 8.8.8.8 icmp\n+ kubectl exec ovn-central-5bc494cb5-np9hm -n kube-ovn -- ovn-trace --ct=new ovn-default 'inport == \"ds1-l6n7p.default\" &amp;&amp; ip.ttl == 64 &amp;&amp; icmp &amp;&amp; eth.src == 0a:00:00:10:00:05 &amp;&amp; ip4.src == 10.16.0.4 &amp;&amp; eth.dst == 00:00:00:B8:CA:43 &amp;&amp; ip4.dst == 8.8.8.8'\n# icmp,reg14=0xf,vlan_tci=0x0000,dl_src=0a:00:00:10:00:05,dl_dst=00:00:00:b8:ca:43,nw_src=10.16.0.4,nw_dst=8.8.8.8,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=0,icmp_code=0\n\ningress(dp=\"ovn-default\", inport=\"ds1-l6n7p.default\")\n-----------------------------------------------------\n 0. ls_in_port_sec_l2 (ovn-northd.c:4143): inport == \"ds1-l6n7p.default\" &amp;&amp; eth.src == {0a:00:00:10:00:05}, priority 50, uuid 39453393\nnext;\n1. ls_in_port_sec_ip (ovn-northd.c:2898): inport == \"ds1-l6n7p.default\" &amp;&amp; eth.src == 0a:00:00:10:00:05 &amp;&amp; ip4.src == {10.16.0.4}, priority 90, uuid 81bcd485\n    next;\n3. ls_in_pre_acl (ovn-northd.c:3269): ip, priority 100, uuid 7b4f4971\n    reg0[0] = 1;\nnext;\n5. ls_in_pre_stateful (ovn-northd.c:3396): reg0[0] == 1, priority 100, uuid 36cdd577\n    ct_next;\n\nct_next(ct_state=new|trk)\n-------------------------\n 6. ls_in_acl (ovn-northd.c:3759): ip &amp;&amp; (!ct.est || (ct.est &amp;&amp; ct_label.blocked == 1)), priority 1, uuid 7608af5b\n    reg0[1] = 1;\nnext;\n10. ls_in_stateful (ovn-northd.c:3995): reg0[1] == 1, priority 100, uuid 2aba1b90\n    ct_commit(ct_label=0/0x1);\nnext;\n16. ls_in_l2_lkup (ovn-northd.c:4470): eth.dst == 00:00:00:b8:ca:43, priority 50, uuid 5c9c3c9f\n    outport = \"ovn-default-ovn-cluster\";\noutput;\n\n...\n</code></pre> <p>\u82e5 trace \u5bf9\u8c61\u4e3a\u8fd0\u884c\u4e8e Underlay \u7f51\u7edc\u4e0b\u7684\u865a\u62df\u673a\uff0c\u9700\u8981\u6dfb\u52a0\u989d\u5916\u53c2\u6570\u6765\u6307\u5b9a\u76ee\u7684 Mac \u5730\u5740\uff1a</p> <pre><code>kubectl ko trace default/virt-handler-7lvml 8.8.8.8 82:7c:9f:83:8c:01 icmp\n</code></pre>"},{"location":"ops/kubectl-ko/#diagnose-allnodesubnetipports-nodenamesubnetnameproto1-ip1-port1proto2-ip2-port2","title":"diagnose {all|node|subnet|IPPorts} [nodename|subnetName|{proto1}-{IP1}-{Port1},{proto2}-{IP2}-{Port2}]","text":"<p>\u8bca\u65ad\u96c6\u7fa4\u7f51\u7edc\u7ec4\u4ef6\u72b6\u6001\uff0c\u5e76\u53bb\u5bf9\u5e94\u8282\u70b9\u7684 <code>kube-ovn-pinger</code> \u68c0\u6d4b\u5f53\u524d\u8282\u70b9\u5230\u5176\u4ed6\u8282\u70b9\u548c\u5173\u952e\u670d\u52a1\u7684\u8fde\u901a\u6027\u548c\u7f51\u7edc\u5ef6\u8fdf\uff1a</p> <pre><code># kubectl ko diagnose all\nswitch c7cd17e8-ceee-4a91-9bb3-e5a313fe1ece (snat)\nport snat-ovn-cluster\n        type: router\n        router-port: ovn-cluster-snat\nswitch 20e0c6d0-023a-4756-aec5-200e0c60f95d (join)\nport node-liumengxin-ovn3-192.168.137.178\n        addresses: [\"00:00:00:64:FF:A8 100.64.0.4\"]\nport node-liumengxin-ovn1-192.168.137.176\n        addresses: [\"00:00:00:AF:98:62 100.64.0.2\"]\nport join-ovn-cluster\n        type: router\n        router-port: ovn-cluster-join\nswitch 0191705c-f827-427b-9de3-3c3b7d971ba5 (central)\nport central-ovn-cluster\n        type: router\n        router-port: ovn-cluster-central\nswitch 2a45ff05-388d-4f85-9daf-e6fccd5833dc (ovn-default)\nport ovn-default-ovn-cluster\n        type: router\n        router-port: ovn-cluster-ovn-default\n    port prometheus-k8s-1.monitoring\n        addresses: [\"00:00:00:AA:37:DF 10.16.0.23\"]\nrouter 212f73dd-d63d-4d72-864b-a537e9afbee1 (ovn-cluster)\nport ovn-cluster-snat\n        mac: \"00:00:00:7A:82:8F\"\nnetworks: [\"172.22.0.1/16\"]\nport ovn-cluster-join\n        mac: \"00:00:00:F8:18:5A\"\nnetworks: [\"100.64.0.1/16\"]\nport ovn-cluster-central\n        mac: \"00:00:00:4D:8C:F5\"\nnetworks: [\"192.101.0.1/16\"]\nport ovn-cluster-ovn-default\n        mac: \"00:00:00:A3:F8:18\"\nnetworks: [\"10.16.0.1/16\"]\nRouting Policies\n     31000                            ip4.dst == 10.16.0.0/16           allow\n     31000                           ip4.dst == 100.64.0.0/16           allow\n     30000                         ip4.dst == 192.168.137.177         reroute                100.64.0.3\n     30000                         ip4.dst == 192.168.137.178         reroute                100.64.0.4\n     29000                 ip4.src == $ovn.default.fake.6_ip4         reroute               100.64.0.22\n     29000                 ip4.src == $ovn.default.fake.7_ip4         reroute               100.64.0.21\n     29000                 ip4.src == $ovn.default.fake.8_ip4         reroute               100.64.0.23\n     29000 ip4.src == $ovn.default.liumengxin.ovn3.192.168.137.178_ip4         reroute                100.64.0.4\n     20000 ip4.src == $ovn.default.liumengxin.ovn1.192.168.137.176_ip4 &amp;&amp; ip4.dst != $ovn.cluster.overlay.subnets.IPv4         reroute                100.64.0.2\n     20000 ip4.src == $ovn.default.liumengxin.ovn2.192.168.137.177_ip4 &amp;&amp; ip4.dst != $ovn.cluster.overlay.subnets.IPv4         reroute                100.64.0.3\n     20000 ip4.src == $ovn.default.liumengxin.ovn3.192.168.137.178_ip4 &amp;&amp; ip4.dst != $ovn.cluster.overlay.subnets.IPv4         reroute                100.64.0.4\nIPv4 Routes\nRoute Table &lt;main&gt;:\n                0.0.0.0/0                100.64.0.1 dst-ip\nUUID                                    LB                  PROTO      VIP                     IPs\ne9bcfd9d-793e-4431-9073-6dec96b75d71    cluster-tcp-load    tcp        10.100.209.132:10660    192.168.137.176:10660\n                                                            tcp        10.101.239.192:6641     192.168.137.177:6641\n                                                            tcp        10.101.240.101:3000     10.16.0.7:3000\n                                                            tcp        10.103.184.186:6642     192.168.137.177:6642\n35d2b7a5-e3a7-485a-a4b7-b4970eb0e63b    cluster-tcp-sess    tcp        10.100.158.128:8080     10.16.0.10:8080,10.16.0.5:8080,10.16.63.30:8080\n                                                            tcp        10.107.26.215:8080      10.16.0.19:8080,10.16.0.20:8080,10.16.0.21:8080\n                                                            tcp        10.107.26.215:9093      10.16.0.19:9093,10.16.0.20:9093,10.16.0.21:9093\n                                                            tcp        10.98.187.99:8080       10.16.0.22:8080,10.16.0.23:8080\n                                                            tcp        10.98.187.99:9090       10.16.0.22:9090,10.16.0.23:9090\nf43303e4-89aa-4d3e-a3dc-278a552fe27b    cluster-udp-load    udp        10.96.0.10:53           10.16.0.4:53,10.16.0.9:53\n_uuid               : 06776304-5a96-43ed-90c4-c4854c251699\naddresses           : []\nexternal_ids        : {vendor=kube-ovn}\nname                : node_liumengxin_ovn2_192.168.137.177_underlay_v6\n\n_uuid               : 62690625-87d5-491c-8675-9fd83b1f433c\naddresses           : []\nexternal_ids        : {vendor=kube-ovn}\nname                : node_liumengxin_ovn1_192.168.137.176_underlay_v6\n\n_uuid               : b03a9bae-94d5-4562-b34c-b5f6198e180b\naddresses           : [\"10.16.0.0/16\", \"100.64.0.0/16\", \"172.22.0.0/16\", \"192.101.0.0/16\"]\nexternal_ids        : {vendor=kube-ovn}\nname                : ovn.cluster.overlay.subnets.IPv4\n\n_uuid               : e1056f3a-24cc-4666-8a91-75ee6c3c2426\naddresses           : []\nexternal_ids        : {vendor=kube-ovn}\nname                : ovn.cluster.overlay.subnets.IPv6\n\n_uuid               : 3e5d5fff-e670-47b2-a2f5-a39f4698a8c5\naddresses           : []\nexternal_ids        : {vendor=kube-ovn}\nname                : node_liumengxin_ovn3_192.168.137.178_underlay_v6\n_uuid               : 2d85dbdc-d0db-4abe-b19e-cc806d32b492\naction              : drop\ndirection           : from-lport\nexternal_ids        : {}\nlabel               : 0\nlog                 : false\nmatch               : \"inport==@ovn.sg.kubeovn_deny_all &amp;&amp; ip\"\nmeter               : []\nname                : []\noptions             : {}\npriority            : 2003\nseverity            : []\n\n_uuid               : de790cc8-f155-405f-bb32-5a51f30c545f\naction              : drop\ndirection           : to-lport\nexternal_ids        : {}\nlabel               : 0\nlog                 : false\nmatch               : \"outport==@ovn.sg.kubeovn_deny_all &amp;&amp; ip\"\nmeter               : []\nname                : []\noptions             : {}\npriority            : 2003\nseverity            : []\nChassis \"e15ed4d4-1780-4d50-b09e-ea8372ed48b8\"\nhostname: liumengxin-ovn1-192.168.137.176\n    Encap stt\n        ip: \"192.168.137.176\"\noptions: {csum=\"true\"}\nPort_Binding node-liumengxin-ovn1-192.168.137.176\n    Port_Binding perf-6vxkn.default\n    Port_Binding kube-state-metrics-5d6885d89-4nf8h.monitoring\n    Port_Binding alertmanager-main-0.monitoring\n    Port_Binding kube-ovn-pinger-6ftdf.kube-system\n    Port_Binding fake-kubelet-67c55dfd89-pv86k.kube-system\n    Port_Binding prometheus-k8s-0.monitoring\nChassis \"eef07da1-f8ad-4775-b14d-bd6a3b4eb0d5\"\nhostname: liumengxin-ovn3-192.168.137.178\n    Encap stt\n        ip: \"192.168.137.178\"\noptions: {csum=\"true\"}\nPort_Binding kube-ovn-pinger-7twb4.kube-system\n    Port_Binding prometheus-adapter-86df476d87-rl88g.monitoring\n    Port_Binding prometheus-k8s-1.monitoring\n    Port_Binding node-liumengxin-ovn3-192.168.137.178\n    Port_Binding perf-ff475.default\n    Port_Binding alertmanager-main-1.monitoring\n    Port_Binding blackbox-exporter-676d976865-tvsjd.monitoring\nChassis \"efa253c9-494d-4719-83ae-b48ab0f11c03\"\nhostname: liumengxin-ovn2-192.168.137.177\n    Encap stt\n        ip: \"192.168.137.177\"\noptions: {csum=\"true\"}\nPort_Binding grafana-6c4c6b8fb7-pzd2c.monitoring\n    Port_Binding node-liumengxin-ovn2-192.168.137.177\n    Port_Binding alertmanager-main-2.monitoring\n    Port_Binding coredns-6789c94dd8-9jqsz.kube-system\n    Port_Binding coredns-6789c94dd8-25d4r.kube-system\n    Port_Binding prometheus-operator-7bbc99fc8b-wgjm4.monitoring\n    Port_Binding prometheus-adapter-86df476d87-gdxmc.monitoring\n    Port_Binding perf-fjnws.default\n    Port_Binding kube-ovn-pinger-vh2xg.kube-system\nds kube-proxy ready\nkube-proxy ready\ndeployment ovn-central ready\ndeployment kube-ovn-controller ready\nds kube-ovn-cni ready\nds ovs-ovn ready\ndeployment coredns ready\novn-nb leader check ok\novn-sb leader check ok\novn-northd leader check ok\n### kube-ovn-controller recent log\n\n### start to diagnose node liumengxin-ovn1-192.168.137.176\n#### ovn-controller log:\n2022-06-03T00:56:44.897Z|16722|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T01:06:44.912Z|16723|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T01:16:44.925Z|16724|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T01:26:44.936Z|16725|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T01:36:44.959Z|16726|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T01:46:44.974Z|16727|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T01:56:44.988Z|16728|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T02:06:45.001Z|16729|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T02:16:45.025Z|16730|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T02:26:45.040Z|16731|inc_proc_eng|INFO|User triggered force recompute.\n\n#### ovs-vswitchd log:\n2022-06-02T23:03:00.137Z|00079|dpif(handler1)|WARN|system@ovs-system: execute ct(commit,zone=14,label=0/0x1,nat(src)),8 failed (Invalid argument) on packet icmp,vlan_tci=0x0000,dl_src=00:00:00:f8:07:c8,dl_dst=00:00:00:fa:1e:50,nw_src=10.16.0.5,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0 icmp_csum:f9d1\n with metadata skb_priority(0),tunnel(tun_id=0x160017000004,src=192.168.137.177,dst=192.168.137.176,ttl=64,tp_src=38881,tp_dst=7471,flags(csum|key)),skb_mark(0),ct_state(0x21),ct_zone(0xe),ct_tuple4(src=10.16.0.5,dst=10.16.0.10,proto=1,tp_src=8,tp_dst=0),in_port(4) mtu 0\n2022-06-02T23:23:31.840Z|00080|dpif(handler1)|WARN|system@ovs-system: execute ct(commit,zone=14,label=0/0x1,nat(src)),8 failed (Invalid argument) on packet icmp,vlan_tci=0x0000,dl_src=00:00:00:f8:07:c8,dl_dst=00:00:00:fa:1e:50,nw_src=10.16.0.5,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0 icmp_csum:15b2\n with metadata skb_priority(0),tunnel(tun_id=0x160017000004,src=192.168.137.177,dst=192.168.137.176,ttl=64,tp_src=38881,tp_dst=7471,flags(csum|key)),skb_mark(0),ct_state(0x21),ct_zone(0xe),ct_tuple4(src=10.16.0.5,dst=10.16.0.10,proto=1,tp_src=8,tp_dst=0),in_port(4) mtu 0\n2022-06-03T00:09:15.659Z|00081|dpif(handler1)|WARN|system@ovs-system: execute ct(commit,zone=14,label=0/0x1,nat(src)),8 failed (Invalid argument) on packet icmp,vlan_tci=0x0000,dl_src=00:00:00:dc:e3:63,dl_dst=00:00:00:fa:1e:50,nw_src=10.16.63.30,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0 icmp_csum:e5a5\n with metadata skb_priority(0),tunnel(tun_id=0x150017000004,src=192.168.137.178,dst=192.168.137.176,ttl=64,tp_src=9239,tp_dst=7471,flags(csum|key)),skb_mark(0),ct_state(0x21),ct_zone(0xe),ct_tuple4(src=10.16.63.30,dst=10.16.0.10,proto=1,tp_src=8,tp_dst=0),in_port(4) mtu 0\n2022-06-03T00:30:13.409Z|00064|dpif(handler2)|WARN|system@ovs-system: execute ct(commit,zone=14,label=0/0x1,nat(src)),8 failed (Invalid argument) on packet icmp,vlan_tci=0x0000,dl_src=00:00:00:f8:07:c8,dl_dst=00:00:00:fa:1e:50,nw_src=10.16.0.5,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0 icmp_csum:6b4a\n with metadata skb_priority(0),tunnel(tun_id=0x160017000004,src=192.168.137.177,dst=192.168.137.176,ttl=64,tp_src=38881,tp_dst=7471,flags(csum|key)),skb_mark(0),ct_state(0x21),ct_zone(0xe),ct_tuple4(src=10.16.0.5,dst=10.16.0.10,proto=1,tp_src=8,tp_dst=0),in_port(4) mtu 0\n2022-06-03T02:02:33.832Z|00082|dpif(handler1)|WARN|system@ovs-system: execute ct(commit,zone=14,label=0/0x1,nat(src)),8 failed (Invalid argument) on packet icmp,vlan_tci=0x0000,dl_src=00:00:00:f8:07:c8,dl_dst=00:00:00:fa:1e:50,nw_src=10.16.0.5,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0 icmp_csum:a819\n with metadata skb_priority(0),tunnel(tun_id=0x160017000004,src=192.168.137.177,dst=192.168.137.176,ttl=64,tp_src=38881,tp_dst=7471,flags(csum|key)),skb_mark(0),ct_state(0x21),ct_zone(0xe),ct_tuple4(src=10.16.0.5,dst=10.16.0.10,proto=1,tp_src=8,tp_dst=0),in_port(4) mtu 0\n\n#### ovs-vsctl show results:\n0d4c4675-c9cc-440a-8c1a-878e17f81b88\n    Bridge br-int\n        fail_mode: secure\n        datapath_type: system\n        Port a2c1a8a8b83a_h\n            Interface a2c1a8a8b83a_h\n        Port \"4fa5c4cbb1a5_h\"\nInterface \"4fa5c4cbb1a5_h\"\nPort ovn-eef07d-0\n            Interface ovn-eef07d-0\n                type: stt\n                options: {csum=\"true\", key=flow, remote_ip=\"192.168.137.178\"}\nPort ovn0\n            Interface ovn0\n                type: internal\n        Port \"04d03360e9a0_h\"\nInterface \"04d03360e9a0_h\"\nPort eeb4d9e51b5d_h\n            Interface eeb4d9e51b5d_h\n        Port mirror0\n            Interface mirror0\n                type: internal\n        Port \"8e5d887ccd80_h\"\nInterface \"8e5d887ccd80_h\"\nPort ovn-efa253-0\n            Interface ovn-efa253-0\n                type: stt\n                options: {csum=\"true\", key=flow, remote_ip=\"192.168.137.177\"}\nPort \"17512d5be1f1_h\"\nInterface \"17512d5be1f1_h\"\nPort br-int\n            Interface br-int\n                type: internal\n    ovs_version: \"2.17.2\"\n\n#### pinger diagnose results:\nI0603 10:35:04.349404   17619 pinger.go:19]\n-------------------------------------------------------------------------------\nKube-OVN:\n  Version:       v1.13.0\n  Build:         2022-04-24_08:02:50\n  Commit:        git-73f9d15\n  Go Version:    go1.17.8\n  Arch:          amd64\n-------------------------------------------------------------------------------\nI0603 10:35:04.376797   17619 config.go:166] pinger config is &amp;{KubeConfigFile: KubeClient:0xc000493380 Port:8080 DaemonSetNamespace:kube-system DaemonSetName:kube-ovn-pinger Interval:5 Mode:job ExitCode:0 InternalDNS:kubernetes.default ExternalDNS: NodeName:liumengxin-ovn1-192.168.137.176 HostIP:192.168.137.176 PodName:kube-ovn-pinger-6ftdf PodIP:10.16.0.10 PodProtocols:[IPv4] ExternalAddress: NetworkMode:kube-ovn PollTimeout:2 PollInterval:15 SystemRunDir:/var/run/openvswitch DatabaseVswitchName:Open_vSwitch DatabaseVswitchSocketRemote:unix:/var/run/openvswitch/db.sock DatabaseVswitchFileDataPath:/etc/openvswitch/conf.db DatabaseVswitchFileLogPath:/var/log/openvswitch/ovsdb-server.log DatabaseVswitchFilePidPath:/var/run/openvswitch/ovsdb-server.pid DatabaseVswitchFileSystemIDPath:/etc/openvswitch/system-id.conf ServiceVswitchdFileLogPath:/var/log/openvswitch/ovs-vswitchd.log ServiceVswitchdFilePidPath:/var/run/openvswitch/ovs-vswitchd.pid ServiceOvnControllerFileLogPath:/var/log/ovn/ovn-controller.log ServiceOvnControllerFilePidPath:/var/run/ovn/ovn-controller.pid}\nI0603 10:35:04.449166   17619 exporter.go:75] liumengxin-ovn1-192.168.137.176: exporter connect successfully\nI0603 10:35:04.554011   17619 ovn.go:21] ovs-vswitchd and ovsdb are up\nI0603 10:35:04.651293   17619 ovn.go:33] ovn_controller is up\nI0603 10:35:04.651342   17619 ovn.go:39] start to check port binding\nI0603 10:35:04.749613   17619 ovn.go:135] chassis id is 1d7f3d6c-eec5-4b3c-adca-2969d9cdfd80\nI0603 10:35:04.763487   17619 ovn.go:49] port in sb is [node-liumengxin-ovn1-192.168.137.176 perf-6vxkn.default kube-state-metrics-5d6885d89-4nf8h.monitoring alertmanager-main-0.monitoring kube-ovn-pinger-6ftdf.kube-system fake-kubelet-67c55dfd89-pv86k.kube-system prometheus-k8s-0.monitoring]\nI0603 10:35:04.763583   17619 ovn.go:61] ovs and ovn-sb binding check passed\nI0603 10:35:05.049309   17619 ping.go:259] start to check apiserver connectivity\nI0603 10:35:05.053666   17619 ping.go:268] connect to apiserver success in 4.27ms\nI0603 10:35:05.053786   17619 ping.go:129] start to check pod connectivity\nI0603 10:35:05.249590   17619 ping.go:159] ping pod: kube-ovn-pinger-6ftdf 10.16.0.10, count: 3, loss count 0, average rtt 16.30ms\nI0603 10:35:05.354135   17619 ping.go:159] ping pod: kube-ovn-pinger-7twb4 10.16.63.30, count: 3, loss count 0, average rtt 1.81ms\nI0603 10:35:05.458460   17619 ping.go:159] ping pod: kube-ovn-pinger-vh2xg 10.16.0.5, count: 3, loss count 0, average rtt 1.92ms\nI0603 10:35:05.458523   17619 ping.go:83] start to check node connectivity\n</code></pre> <p>\u5982\u679c diagnose \u7684\u76ee\u6807\u6307\u5b9a\u4e3a subnet \u8be5\u811a\u672c\u4f1a\u5728 subnet \u4e0a\u5efa\u7acb daemonset\uff0c\u7531 <code>kube-ovn-pinger</code> \u53bb\u63a2\u6d4b\u8fd9\u4e2a daemonset \u7684\u6240\u6709 pod \u7684\u8fde\u901a\u6027\u548c\u7f51\u7edc\u5ef6\u65f6\uff0c\u6d4b\u8bd5\u5b8c\u540e\u81ea\u52a8\u9500\u6bc1\u8be5 daemonset\u3002</p> <p>\u5982\u679c diagnose \u7684\u76ee\u6807\u6307\u5b9a\u4e3a IPPorts \u8be5\u811a\u672c\u4f1a\u8ba9\u6bcf\u4e2a <code>kube-ovn-pinger</code> pod \u53bb\u63a2\u6d4b\u76ee\u6807\u534f\u8bae\uff0cIP\uff0cPort \u662f\u5426\u53ef\u8fbe\u3002</p>"},{"location":"ops/kubectl-ko/#tuning-install-fastpathlocal-install-fastpathremove-fastpathinstall-sttlocal-install-sttremove-stt-centos7centos8-kernel-devel-version","title":"tuning {install-fastpath|local-install-fastpath|remove-fastpath|install-stt|local-install-stt|remove-stt} {centos7|centos8}} [kernel-devel-version]","text":"<p>\u8be5\u547d\u4ee4\u6267\u884c\u6027\u80fd\u8c03\u4f18\u76f8\u5173\u64cd\u4f5c\uff0c\u5177\u4f53\u4f7f\u7528\u8bf7\u53c2\u8003\u6027\u80fd\u8c03\u4f18\u3002</p>"},{"location":"ops/kubectl-ko/#reload","title":"reload","text":"<p>\u8be5\u547d\u4ee4\u91cd\u542f\u6240\u6709 Kube-OVN \u76f8\u5173\u7ec4\u4ef6\uff1a</p> <pre><code># kubectl ko reload\npod \"ovn-central-8684dd94bd-vzgcr\" deleted\nWaiting for deployment \"ovn-central\" rollout to finish: 0 of 1 updated replicas are available...\ndeployment \"ovn-central\" successfully rolled out\npod \"ovs-ovn-bsnvz\" deleted\npod \"ovs-ovn-m9b98\" deleted\npod \"kube-ovn-controller-8459db5ff4-64c62\" deleted\nWaiting for deployment \"kube-ovn-controller\" rollout to finish: 0 of 1 updated replicas are available...\ndeployment \"kube-ovn-controller\" successfully rolled out\npod \"kube-ovn-cni-2klnh\" deleted\npod \"kube-ovn-cni-t2jz4\" deleted\nWaiting for daemon set \"kube-ovn-cni\" rollout to finish: 0 of 2 updated pods are available...\nWaiting for daemon set \"kube-ovn-cni\" rollout to finish: 1 of 2 updated pods are available...\ndaemon set \"kube-ovn-cni\" successfully rolled out\npod \"kube-ovn-pinger-ln72z\" deleted\npod \"kube-ovn-pinger-w8lrk\" deleted\nWaiting for daemon set \"kube-ovn-pinger\" rollout to finish: 0 of 2 updated pods are available...\nWaiting for daemon set \"kube-ovn-pinger\" rollout to finish: 1 of 2 updated pods are available...\ndaemon set \"kube-ovn-pinger\" successfully rolled out\npod \"kube-ovn-monitor-7fb67d5488-7q6zb\" deleted\nWaiting for deployment \"kube-ovn-monitor\" rollout to finish: 0 of 1 updated replicas are available...\ndeployment \"kube-ovn-monitor\" successfully rolled out\n</code></pre>"},{"location":"ops/kubectl-ko/#log","title":"log","text":"<p>\u4f7f\u7528\u8be5\u547d\u4ee4\u4f1a\u6293\u53d6 kube-ovn \u6240\u6709\u8282\u70b9\u4e0a\u7684 Kube-OVN\uff0cOVN\uff0cOpenvswitch \u7684 log \u4ee5\u53ca linux \u5e38\u7528\u7684\u4e00\u4e9b debug \u4fe1\u606f\u3002</p> <pre><code># kubectl ko log all\nCollecting kube-ovn logging files\nCollecting ovn logging files\nCollecting openvswitch logging files\nCollecting linux dmesg files\nCollecting linux iptables-legacy files\nCollecting linux iptables-nft files\nCollecting linux route files\nCollecting linux link files\nCollecting linux neigh files\nCollecting linux memory files\nCollecting linux top files\nCollecting linux sysctl files\nCollecting linux netstat files\nCollecting linux addr files\nCollecting linux ipset files\nCollecting linux tcp files\nCollected files have been saved in the directory /root/kubectl-ko-log\n</code></pre> <p>\u76ee\u5f55\u5982\u4e0b\uff1a</p> <pre><code># tree kubectl-ko-log/\nkubectl-ko-log/\n|-- kube-ovn-control-plane\n|   |-- kube-ovn\n|   |   |-- kube-ovn-cni.log\n|   |   |-- kube-ovn-monitor.log\n|   |   `-- kube-ovn-pinger.log\n|   |-- linux\n|   |   |-- addr.log\n|   |   |-- dmesg.log\n|   |   |-- ipset.log\n|   |   |-- iptables-legacy.log\n|   |   |-- iptables-nft.log\n|   |   |-- link.log\n|   |   |-- memory.log\n|   |   |-- neigh.log\n|   |   |-- netstat.log\n|   |   |-- route.log\n|   |   |-- sysctl.log\n|   |   |-- tcp.log\n|   |   `-- top.log\n|   |-- openvswitch\n|   |   |-- ovs-vswitchd.log\n|   |   `-- ovsdb-server.log\n|   `-- ovn\n|       |-- ovn-controller.log\n|       |-- ovn-northd.log\n|       |-- ovsdb-server-nb.log\n|       `-- ovsdb-server-sb.log\n</code></pre>"},{"location":"ops/kubectl-ko/#perf-image","title":"perf [image]","text":"<p>\u8be5\u547d\u4ee4\u4f1a\u53bb\u6d4b\u8bd5 Kube-OVN \u7684\u4e00\u4e9b\u6027\u80fd\u6307\u6807\u5982\u4e0b\uff1a</p> <ol> <li>\u5bb9\u5668\u7f51\u7edc\u7684\u6027\u80fd\u6307\u6807\uff1b</li> <li>Hostnetwork \u7f51\u7edc\u6027\u80fd\u6307\u6807\uff1b</li> <li>\u5bb9\u5668\u7f51\u7edc\u7ec4\u64ad\u62a5\u6587\u6027\u80fd\u6307\u6807\uff1b</li> <li>OVN-NB, OVN-SB, OVN-Northd leader \u5220\u9664\u6062\u590d\u6240\u9700\u65f6\u95f4\u3002</li> </ol> <p>\u53c2\u6570 image \u7528\u4e8e\u6307\u5b9a\u6027\u80fd\u6d4b\u8bd5 pod \u6240\u7528\u7684\u955c\u50cf\uff0c\u9ed8\u8ba4\u60c5\u51b5\u4e0b\u662f <code>kubeovn/test:v1.12.0</code>, \u8bbe\u7f6e\u8be5\u53c2\u6570\u4e3b\u8981\u662f\u4e3a\u4e86\u79bb\u7ebf\u573a\u666f\uff0c\u5c06\u955c\u50cf\u62c9\u5230\u5185\u7f51\u73af\u5883\u53ef\u80fd\u4f1a\u6709\u955c\u50cf\u540d\u53d8\u5316\u3002</p> <pre><code># kubectl ko perf\n============================== Prepareing Performance Test Resources ===============================\npod/test-client created\npod/test-host-client created\npod/test-server created\npod/test-host-server created\nservice/test-server created\npod/test-client condition met\npod/test-host-client condition met\npod/test-host-server condition met\npod/test-server condition met\n====================================================================================================\n============================ Start Pod Network Unicast Performance Test ============================\nSize            TCP Latency     TCP Bandwidth   UDP Latency     UDP Lost Rate   UDP Bandwidth\n64              82.8 us         97.7 Mbits/sec  67.6 us         (0%)            8.42 Mbits/sec\n128             85.4 us         167 Mbits/sec   67.2 us         (0%)            17.2 Mbits/sec\n512             85.8 us         440 Mbits/sec   68.7 us         (0%)            68.4 Mbits/sec\n1k              85.1 us         567 Mbits/sec   68.7 us         (0%)            134 Mbits/sec\n4k              138 us          826 Mbits/sec   78.1 us         (1.4%)          503 Mbits/sec\n====================================================================================================\n=============================== Start Host Network Performance Test ================================\nSize            TCP Latency     TCP Bandwidth   UDP Latency     UDP Lost Rate   UDP Bandwidth\n64              49.7 us         120 Mbits/sec   37.9 us         (0%)            18.6 Mbits/sec\n128             49.7 us         200 Mbits/sec   38.1 us         (0%)            35.5 Mbits/sec\n512             51.9 us         588 Mbits/sec   38.9 us         (0%)            142 Mbits/sec\n1k              51.7 us         944 Mbits/sec   37.2 us         (0%)            279 Mbits/sec\n4k              74.9 us         1.66 Gbits/sec  39.9 us         (0%)            1.20 Gbits/sec\n====================================================================================================\n============================== Start Service Network Performance Test ==============================\nSize            TCP Latency     TCP Bandwidth   UDP Latency     UDP Lost Rate   UDP Bandwidth\n64              111 us          96.3 Mbits/sec  88.4 us         (0%)            7.59 Mbits/sec\n128             83.7 us         150 Mbits/sec   69.2 us         (0%)            16.9 Mbits/sec\n512             87.4 us         374 Mbits/sec   75.8 us         (0%)            60.9 Mbits/sec\n1k              88.2 us         521 Mbits/sec   73.1 us         (0%)            123 Mbits/sec\n4k              148 us          813 Mbits/sec   77.6 us         (0.0044%)       451 Mbits/sec\n====================================================================================================\n=========================== Start Pod Multicast Network Performance Test ===========================\nSize            UDP Latency     UDP Lost Rate   UDP Bandwidth\n64              0.014 ms        (0.17%)         5.80 Mbits/sec\n128             0.012 ms        (0%)            11.4 Mbits/sec\n512             0.016 ms        (0%)            46.1 Mbits/sec\n1k              0.023 ms        (0.073%)        89.8 Mbits/sec\n4k              0.035 ms        (1.3%)          126 Mbits/sec\n====================================================================================================\n============================= Start Host Multicast Network Performance =============================\nSize            UDP Latency     UDP Lost Rate   UDP Bandwidth\n64              0.007 ms        (0%)            9.95 Mbits/sec\n128             0.005 ms        (0%)            21.8 Mbits/sec\n512             0.008 ms        (0%)            86.8 Mbits/sec\n1k              0.013 ms        (0.045%)        168 Mbits/sec\n4k              0.010 ms        (0.31%)         242 Mbits/sec\n====================================================================================================\n================================== Start Leader Recover Time Test ==================================\nDelete ovn central nb pod\npod \"ovn-central-5cb9c67d75-tlz9w\" deleted\nWaiting for ovn central nb pod running\n=============================== OVN nb Recovery takes 3.305236803 s ================================\nDelete ovn central sb pod\npod \"ovn-central-5cb9c67d75-szx4c\" deleted\nWaiting for ovn central sb pod running\n=============================== OVN sb Recovery takes 3.462698535 s ================================\nDelete ovn central northd pod\npod \"ovn-central-5cb9c67d75-zqmqv\" deleted\nWaiting for ovn central northd pod running\n============================= OVN northd Recovery takes 2.691291403 s ==============================\n====================================================================================================\n================================= Remove Performance Test Resource =================================\nrm -f unicast-test-client.log\nrm -f unicast-test-host-client.log\nrm -f unicast-test-client.log\nkubectl ko nbctl lb-del test-server\nrm -f multicast-test-server.log\nkubectl exec ovs-ovn-gxdrf -n kube-system -- ip maddr del 01:00:5e:00:00:64 dev eth0\nkubectl exec ovs-ovn-h57bf -n kube-system -- ip maddr del 01:00:5e:00:00:64 dev eth0\nrm -f multicast-test-host-server.log\npod \"test-client\" deleted\npod \"test-host-client\" deleted\npod \"test-host-server\" deleted\npod \"test-server\" deleted\nservice \"test-server\" deleted\n====================================================================================================\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"ops/recover-db/","title":"OVN \u6570\u636e\u5e93\u5907\u4efd\u548c\u6062\u590d","text":"<p>\u672c\u6587\u6863\u4ecb\u7ecd\u5982\u4f55\u8fdb\u884c\u6570\u636e\u5e93\u5907\u4efd\uff0c\u4ee5\u53ca\u5728\u4e0d\u540c\u60c5\u51b5\u4e0b\u5982\u4f55\u901a\u8fc7\u5df2\u6709\u7684\u6570\u636e\u5e93\u6587\u4ef6\u8fdb\u884c\u96c6\u7fa4\u6062\u590d\u3002</p>"},{"location":"ops/recover-db/#_1","title":"\u6570\u636e\u5e93\u5907\u4efd","text":"<p>\u5229\u7528 kubectl \u63d2\u4ef6\u7684 backup \u547d\u4ee4\u53ef\u4ee5\u5bf9\u6570\u636e\u5e93\u6587\u4ef6\u8fdb\u884c\u5907\u4efd\uff0c\u4ee5\u7528\u4e8e\u6545\u969c\u65f6\u6062\u590d\uff1a</p> <pre><code># kubectl ko nb backup\ntar: Removing leading `/' from member names\nbackup ovn-nb db to /root/ovnnb_db.060223191654183154.backup\n\n# kubectl ko sb backup\ntar: Removing leading `/' from member names\nbackup ovn-nb db to /root/ovnsb_db.060223191654183154.backup\n</code></pre>"},{"location":"ops/recover-db/#_2","title":"\u96c6\u7fa4\u90e8\u5206\u6545\u969c\u6062\u590d","text":"<p>\u82e5\u96c6\u7fa4\u4e2d\u5b58\u5728\u90e8\u5206\u8282\u70b9\u56e0\u4e3a\u65ad\u7535\uff0c\u6587\u4ef6\u7cfb\u7edf\u6545\u969c\u6216\u78c1\u76d8\u7a7a\u95f4\u4e0d\u8db3\u5bfc\u81f4\u5de5\u4f5c\u5f02\u5e38\uff0c \u4f46\u662f\u96c6\u7fa4\u4ecd\u53ef\u6b63\u5e38\u5de5\u4f5c\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0b\u6b65\u9aa4\u8fdb\u884c\u6062\u590d\u3002</p>"},{"location":"ops/recover-db/#_3","title":"\u67e5\u770b\u65e5\u5fd7\u786e\u8ba4\u72b6\u6001\u5f02\u5e38","text":"<p>\u67e5\u770b\u5bf9\u5e94\u8282\u70b9 <code>/var/log/ovn/ovn-northd.log</code>\uff0c\u82e5\u63d0\u793a\u7c7b\u4f3c\u9519\u8bef\u5219\u53ef\u5224\u65ad\u6570\u636e\u5e93\u5b58\u5728\u5f02\u5e38</p> <pre><code> * ovn-northd is not running\novsdb-server: ovsdb error: error reading record 2739 from OVN_Northbound log: record 2739 advances commit index to 6308 but last log index is 6307\n* Starting ovsdb-nb\n</code></pre>"},{"location":"ops/recover-db/#_4","title":"\u4ece\u96c6\u7fa4\u4e2d\u8e22\u51fa\u5bf9\u5e94\u8282\u70b9","text":"<p>\u6839\u636e\u65e5\u5fd7\u63d0\u793a\u662f <code>OVN_Northbound</code> \u8fd8\u662f <code>OVN_Southbound</code> \u9009\u62e9\u5bf9\u5e94\u7684\u6570\u636e\u5e93\u8fdb\u884c\u64cd\u4f5c\u3002 \u4e0a\u8ff0\u65e5\u5fd7\u63d0\u793a\u4e3a <code>OVN_Northbound</code> \u5219\u5bf9 ovn-nb \u8fdb\u884c\u64cd\u4f5c\uff1a</p> <pre><code># kubectl ko nb status\n9182\nName: OVN_Northbound\nCluster ID: e75f (e75fa340-49ed-45ab-990e-26cb865ebc85)\nServer ID: 9182 (9182e8dd-b5b0-4dd8-8518-598cc1e374f3)\nAddress: tcp:[10.0.128.61]:6643\nStatus: cluster member\nRole: leader\nTerm: 1454\nLeader: self\nVote: self\n\nLast Election started 1732603 ms ago, reason: timeout\nLast Election won: 1732587 ms ago\nElection timer: 1000\nLog: [7332, 12512]\nEntries not yet committed: 1\nEntries not yet applied: 1\nConnections: -&gt;f080 &lt;-f080 &lt;-e631 -&gt;e631\nDisconnections: 1\nServers:\n    f080 (f080 at tcp:[10.0.129.139]:6643) next_index=12512 match_index=12510 last msg 63 ms ago\n    9182 (9182 at tcp:[10.0.128.61]:6643) (self) next_index=10394 match_index=12510\ne631 (e631 at tcp:[10.0.131.173]:6643) next_index=12512 match_index=0\n</code></pre> <p>\u4ece\u96c6\u7fa4\u4e2d\u8e22\u51fa\u72b6\u6001\u5f02\u5e38\u8282\u70b9\uff1a</p> <pre><code>kubectl ko nb kick e631\n</code></pre> <p>\u767b\u5f55\u5f02\u5e38\u8282\u70b9\uff0c\u5220\u9664\u5bf9\u5e94\u7684\u6570\u636e\u5e93\u6587\u4ef6\uff1a</p> <pre><code>mv /etc/origin/ovn/ovnnb_db.db /tmp\n</code></pre> <p>\u5220\u9664\u5bf9\u5e94\u8282\u70b9\u7684 <code>ovn-central</code> Pod\uff0c\u7b49\u5f85\u96c6\u7fa4\u81ea\u52a8\u6062\u590d\uff1a</p> <pre><code>kubectl delete pod -n kube-system ovn-central-xxxx\n</code></pre>"},{"location":"ops/recover-db/#_5","title":"\u96c6\u7fa4\u4e0d\u80fd\u6b63\u5e38\u5de5\u4f5c\u4e0b\u7684\u6062\u590d","text":"<p>\u82e5\u96c6\u7fa4\u591a\u6570\u8282\u70b9\u53d7\u635f\u65e0\u6cd5\u9009\u4e3e\u51fa leader\uff0c\u8bf7\u53c2\u7167\u4e0b\u9762\u7684\u6b65\u9aa4\u8fdb\u884c\u6062\u590d\u3002</p>"},{"location":"ops/recover-db/#ovn-central","title":"\u505c\u6b62 ovn-central","text":"<p>\u8bb0\u5f55\u5f53\u524d <code>ovn-central</code> \u526f\u672c\u6570\u91cf\uff0c\u5e76\u505c\u6b62 <code>ovn-central</code> \u907f\u514d\u65b0\u7684\u6570\u636e\u5e93\u53d8\u66f4\u5f71\u54cd\u6062\u590d\uff1a</p> <pre><code>kubectl scale deployment -n kube-system ovn-central --replicas=0\n</code></pre>"},{"location":"ops/recover-db/#_6","title":"\u9009\u62e9\u5907\u4efd","text":"<p>\u7531\u4e8e\u591a\u6570\u8282\u70b9\u53d7\u635f\uff0c\u9700\u8981\u4ece\u67d0\u4e2a\u6570\u636e\u5e93\u6587\u4ef6\u8fdb\u884c\u6062\u590d\u91cd\u5efa\u96c6\u7fa4\u3002\u5982\u679c\u4e4b\u524d\u5907\u4efd\u8fc7\u6570\u636e\u5e93 \u53ef\u4f7f\u7528\u4e4b\u524d\u7684\u5907\u4efd\u6587\u4ef6\u8fdb\u884c\u6062\u590d\u3002\u5982\u679c\u6ca1\u6709\u8fdb\u884c\u8fc7\u5907\u4efd\u53ef\u4ee5\u4f7f\u7528\u4e0b\u9762\u7684\u6b65\u9aa4\u4ece\u5df2\u6709\u7684\u6570\u636e\u5e93\u6587\u4ef6 \u4e2d\u751f\u6210\u4e00\u4e2a\u5907\u4efd\u3002</p> <p>\u7531\u4e8e\u9ed8\u8ba4\u6587\u4ef6\u5939\u4e0b\u7684\u6570\u636e\u5e93\u6587\u4ef6\u4e3a\u96c6\u7fa4\u683c\u5f0f\u6570\u636e\u5e93\u6587\u4ef6\uff0c\u5305\u542b\u5f53\u524d\u96c6\u7fa4\u7684\u4fe1\u606f\uff0c\u65e0\u6cd5\u76f4\u63a5 \u7528\u8be5\u6587\u4ef6\u91cd\u5efa\u6570\u636e\u5e93\uff0c\u9700\u8981\u4f7f\u7528 <code>ovsdb-tool cluster-to-standalone</code> \u8fdb\u884c\u683c\u5f0f\u8f6c\u6362\u3002</p> <p>\u9009\u62e9 <code>ovn-central</code> \u73af\u5883\u53d8\u91cf <code>NODE_IPS</code> \u4e2d\u6392\u7b2c\u4e00\u7684\u8282\u70b9\u6062\u590d\u6570\u636e\u5e93\u6587\u4ef6\uff0c \u5982\u679c\u7b2c\u4e00\u4e2a\u8282\u70b9\u6570\u636e\u5e93\u6587\u4ef6\u5df2\u635f\u574f\uff0c\u4ece\u5176\u4ed6\u673a\u5668 <code>/etc/origin/ovn</code> \u4e0b\u590d\u5236\u6587\u4ef6\u5230\u7b2c\u4e00\u53f0\u673a\u5668 \uff0c \u6267\u884c\u4e0b\u5217\u547d\u4ee4\u751f\u6210\u6570\u636e\u5e93\u6587\u4ef6\u5907\u4efd\u3002</p> <pre><code>docker run -it -v /etc/origin/ovn:/etc/ovn kubeovn/kube-ovn:v1.13.0 bash\ncd /etc/ovn/\novsdb-tool cluster-to-standalone ovnnb_db_standalone.db ovnnb_db.db\novsdb-tool cluster-to-standalone ovnsb_db_standalone.db ovnsb_db.db\n</code></pre>"},{"location":"ops/recover-db/#ovn-central_1","title":"\u5220\u9664\u6bcf\u4e2a ovn-central \u8282\u70b9\u4e0a\u7684\u6570\u636e\u5e93\u6587\u4ef6","text":"<p>\u4e3a\u4e86\u907f\u514d\u91cd\u5efa\u96c6\u7fa4\u65f6\u4f7f\u7528\u5230\u9519\u8bef\u7684\u6570\u636e\uff0c\u9700\u8981\u5bf9\u5df2\u6709\u6570\u636e\u5e93\u6587\u4ef6\u8fdb\u884c\u6e05\u7406\uff1a</p> <pre><code>mv /etc/origin/ovn/ovnnb_db.db /tmp\nmv /etc/origin/ovn/ovnsb_db.db /tmp\n</code></pre>"},{"location":"ops/recover-db/#_7","title":"\u6062\u590d\u6570\u636e\u5e93\u96c6\u7fa4","text":"<p>\u5c06\u5907\u4efd\u6570\u636e\u5e93\u5206\u522b\u91cd\u547d\u540d\u4e3a <code>ovnnb_db.db</code> \u548c <code>ovnsb_db.db</code>\uff0c\u5e76\u590d\u5236\u5230 <code>ovn-central</code>  \u73af\u5883\u53d8\u91cf <code>NODE_IPS</code> \u4e2d\u6392\u7b2c\u4e00\u673a\u5668\u7684 <code>/etc/origin/ovn/</code> \u76ee\u5f55\u4e0b\uff1a</p> <pre><code>mv /etc/origin/ovn/ovnnb_db_standalone.db /etc/origin/ovn/ovnnb_db.db\nmv /etc/origin/ovn/ovnsb_db_standalone.db /etc/origin/ovn/ovnsb_db.db\n</code></pre> <p>\u6062\u590d <code>ovn-central</code> \u7684\u526f\u672c\u6570\uff1a</p> <pre><code>kubectl scale deployment -n kube-system ovn-central --replicas=3\nkubectl rollout status deployment/ovn-central -n kube-system\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"reference/architecture/","title":"\u603b\u4f53\u67b6\u6784","text":"<p>\u672c\u6587\u6863\u5c06\u4ecb\u7ecd Kube-OVN \u7684\u603b\u4f53\u67b6\u6784\uff0c\u548c\u5404\u4e2a\u7ec4\u4ef6\u7684\u529f\u80fd\u4ee5\u53ca\u5176\u4e4b\u95f4\u7684\u4ea4\u4e92\u3002</p> <p>\u603b\u4f53\u6765\u770b\uff0cKube-OVN \u4f5c\u4e3a Kubernetes \u548c OVN \u4e4b\u95f4\u7684\u4e00\u4e2a\u6865\u6881\uff0c\u5c06\u6210\u719f\u7684 SDN \u548c\u4e91\u539f\u751f\u76f8\u7ed3\u5408\u3002 \u8fd9\u610f\u5473\u7740 Kube-OVN \u4e0d\u4ec5\u901a\u8fc7 OVN \u5b9e\u73b0\u4e86 Kubernetes \u4e0b\u7684\u7f51\u7edc\u89c4\u8303\uff0c\u4f8b\u5982 CNI\uff0cService \u548c Networkpolicy\uff0c\u8fd8\u5c06\u5927\u91cf\u7684 SDN \u9886\u57df\u80fd\u529b\u5e26\u5165\u4e91\u539f\u751f\uff0c\u4f8b\u5982\u903b\u8f91\u4ea4\u6362\u673a\uff0c\u903b\u8f91\u8def\u7531\u5668\uff0cVPC\uff0c\u7f51\u5173\uff0cQoS\uff0cACL \u548c\u6d41\u91cf\u955c\u50cf\u3002</p> <p>\u540c\u65f6 Kube-OVN \u8fd8\u4fdd\u6301\u4e86\u826f\u597d\u7684\u5f00\u653e\u6027\u53ef\u4ee5\u548c\u8bf8\u591a\u6280\u672f\u65b9\u6848\u96c6\u6210\uff0c\u4f8b\u5982 Cilium\uff0cSubmariner\uff0cPrometheus\uff0cKubeVirt \u7b49\u7b49\u3002</p>"},{"location":"reference/architecture/#_2","title":"\u7ec4\u4ef6\u4ecb\u7ecd","text":"<p>Kube-OVN \u7684\u7ec4\u4ef6\u53ef\u4ee5\u5927\u81f4\u5206\u4e3a\u4e09\u7c7b\uff1a</p> <ul> <li>\u4e0a\u6e38 OVN/OVS \u7ec4\u4ef6\u3002</li> <li>\u6838\u5fc3\u63a7\u5236\u5668\u548c Agent\u3002</li> <li>\u76d1\u63a7\uff0c\u8fd0\u7ef4\u5de5\u5177\u548c\u6269\u5c55\u7ec4\u4ef6\u3002</li> </ul> <p></p>"},{"location":"reference/architecture/#ovnovs","title":"\u4e0a\u6e38 OVN/OVS \u7ec4\u4ef6","text":"<p>\u8be5\u7c7b\u578b\u7ec4\u4ef6\u6765\u81ea OVN/OVS \u793e\u533a\uff0c\u5e76\u9488\u5bf9 Kube-OVN \u7684\u4f7f\u7528\u573a\u666f\u505a\u4e86\u7279\u5b9a\u4fee\u6539\u3002 OVN/OVS \u672c\u8eab\u662f\u4e00\u5957\u6210\u719f\u7684\u7ba1\u7406\u865a\u673a\u548c\u5bb9\u5668\u7684 SDN \u7cfb\u7edf\uff0c\u6211\u4eec\u5f3a\u70c8\u5efa\u8bae \u5bf9 Kube-OVN \u5b9e\u73b0\u611f\u5174\u8da3\u7684\u7528\u6237\u5148\u53bb\u8bfb\u4e00\u4e0b ovn-architecture(7) \u6765\u4e86\u89e3\u4ec0\u4e48\u662f OVN \u4ee5\u53ca \u5982\u4f55\u548c\u5b83\u8fdb\u884c\u96c6\u6210\u3002Kube-OVN \u4f7f\u7528 OVN \u7684\u5317\u5411\u63a5\u53e3\u521b\u5efa\u548c\u8c03\u6574\u865a\u62df\u7f51\u7edc\uff0c\u5e76\u5c06\u5176\u4e2d\u7684\u7f51\u7edc\u6982\u5ff5\u6620\u5c04\u5230 Kubernetes \u4e4b\u5185\u3002</p> <p>\u6240\u6709 OVN/OVS \u76f8\u5173\u7ec4\u4ef6\u90fd\u5df2\u6253\u5305\u6210\u5bf9\u5e94\u955c\u50cf\uff0c\u5e76\u53ef\u5728 Kubernetes \u4e2d\u8fd0\u884c\u3002</p>"},{"location":"reference/architecture/#ovn-central","title":"ovn-central","text":"<p><code>ovn-central</code> Deployment \u8fd0\u884c OVN \u7684\u7ba1\u7406\u5e73\u9762\u7ec4\u4ef6\uff0c\u5305\u62ec <code>ovn-nb</code>, <code>ovn-sb</code>, \u548c <code>ovn-northd</code>\u3002</p> <ul> <li><code>ovn-nb</code>\uff1a \u4fdd\u5b58\u865a\u62df\u7f51\u7edc\u914d\u7f6e\uff0c\u5e76\u63d0\u4f9b API \u8fdb\u884c\u865a\u62df\u7f51\u7edc\u7ba1\u7406\u3002<code>kube-ovn-controller</code> \u5c06\u4f1a\u4e3b\u8981\u548c <code>ovn-nb</code> \u8fdb\u884c\u4ea4\u4e92\u914d\u7f6e\u865a\u62df\u7f51\u7edc\u3002</li> <li><code>ovn-sb</code>\uff1a \u4fdd\u5b58\u4ece <code>ovn-nb</code> \u7684\u903b\u8f91\u7f51\u7edc\u751f\u6210\u7684\u903b\u8f91\u6d41\u8868\uff0c\u4ee5\u53ca\u5404\u4e2a\u8282\u70b9\u7684\u5b9e\u9645\u7269\u7406\u7f51\u7edc\u72b6\u6001\u3002</li> <li><code>ovn-northd</code>\uff1a\u5c06 <code>ovn-nb</code> \u7684\u865a\u62df\u7f51\u7edc\u7ffb\u8bd1\u6210 <code>ovn-sb</code> \u4e2d\u7684\u903b\u8f91\u6d41\u8868\u3002</li> </ul> <p>\u591a\u4e2a <code>ovn-central</code> \u5b9e\u4f8b\u4f1a\u901a\u8fc7 Raft \u534f\u8bae\u540c\u6b65\u6570\u636e\u4fdd\u8bc1\u9ad8\u53ef\u7528\u3002</p>"},{"location":"reference/architecture/#ovs-ovn","title":"ovs-ovn","text":"<p><code>ovs-ovn</code> \u4ee5 DaemonSet \u5f62\u5f0f\u8fd0\u884c\u5728\u6bcf\u4e2a\u8282\u70b9\uff0c\u5728 Pod \u5185\u8fd0\u884c\u4e86 <code>openvswitch</code>, <code>ovsdb</code>, \u548c <code>ovn-controller</code>\u3002\u8fd9\u4e9b\u7ec4\u4ef6\u4f5c\u4e3a <code>ovn-central</code> \u7684 Agent \u5c06\u903b\u8f91\u6d41\u8868\u7ffb\u8bd1\u6210\u771f\u5b9e\u7684\u7f51\u7edc\u914d\u7f6e\u3002</p>"},{"location":"reference/architecture/#agent","title":"\u6838\u5fc3\u63a7\u5236\u5668\u548c Agent","text":"<p>\u8be5\u90e8\u5206\u4e3a Kube-OVN \u7684\u6838\u5fc3\u7ec4\u4ef6\uff0c\u4f5c\u4e3a OVN \u548c Kubernetes \u4e4b\u95f4\u7684\u4e00\u4e2a\u6865\u6881\uff0c\u5c06\u4e24\u4e2a\u7cfb\u7edf\u6253\u901a\u5e76\u5c06\u7f51\u7edc\u6982\u5ff5\u8fdb\u884c\u76f8\u4e92\u8f6c\u6362\u3002 \u5927\u90e8\u5206\u7684\u6838\u5fc3\u529f\u80fd\u90fd\u5728\u8be5\u90e8\u5206\u7ec4\u4ef6\u4e2d\u5b9e\u73b0\u3002</p>"},{"location":"reference/architecture/#kube-ovn-controller","title":"kube-ovn-controller","text":"<p>\u8be5\u7ec4\u4ef6\u4e3a\u4e00\u4e2a Deployment \u6267\u884c\u6240\u6709 Kubernetes \u5185\u8d44\u6e90\u5230 OVN \u8d44\u6e90\u7684\u7ffb\u8bd1\u5de5\u4f5c\uff0c\u5176\u4f5c\u7528\u76f8\u5f53\u4e8e\u6574\u4e2a Kube-OVN \u7cfb\u7edf\u7684\u63a7\u5236\u5e73\u9762\u3002 <code>kube-ovn-controller</code> \u76d1\u542c\u4e86\u6240\u6709\u548c\u7f51\u7edc\u529f\u80fd\u76f8\u5173\u8d44\u6e90\u7684\u4e8b\u4ef6\uff0c\u5e76\u6839\u636e\u8d44\u6e90\u53d8\u5316\u60c5\u51b5\u66f4\u65b0 OVN \u5185\u7684\u903b\u8f91\u7f51\u7edc\u3002\u4e3b\u8981\u76d1\u542c\u7684\u8d44\u6e90\u5305\u62ec\uff1a Pod\uff0cService\uff0cEndpoint\uff0cNode\uff0cNetworkPolicy\uff0cVPC\uff0cSubnet\uff0cVlan\uff0cProviderNetwork\u3002</p> <p>\u4ee5 Pod \u4e8b\u4ef6\u4e3a\u4f8b\uff0c <code>kube-ovn-controller</code> \u76d1\u542c\u5230 Pod \u521b\u5efa\u4e8b\u4ef6\u540e\uff0c\u901a\u8fc7\u5185\u7f6e\u7684\u5185\u5b58 IPAM \u529f\u80fd\u5206\u914d\u5730\u5740\uff0c\u5e76\u8c03\u7528 <code>ovn-central</code> \u521b\u5efa \u903b\u8f91\u7aef\u53e3\uff0c\u9759\u6001\u8def\u7531\u548c\u53ef\u80fd\u7684 ACL \u89c4\u5219\u3002\u63a5\u4e0b\u6765 <code>kube-ovn-controller</code> \u5c06\u5206\u914d\u5230\u7684\u5730\u5740\uff0c\u548c\u5b50\u7f51\u4fe1\u606f\u4f8b\u5982 CIDR\uff0c\u7f51\u5173\uff0c\u8def\u7531\u7b49\u4fe1\u606f\u5199\u4f1a\u5230 Pod \u7684 annotation \u4e2d\u3002\u8be5 annotation \u540e\u7eed\u4f1a\u88ab <code>kube-ovn-cni</code> \u8bfb\u53d6\u7528\u6765\u914d\u7f6e\u672c\u5730\u7f51\u7edc\u3002</p>"},{"location":"reference/architecture/#kube-ovn-cni","title":"kube-ovn-cni","text":"<p>\u8be5\u7ec4\u4ef6\u4e3a\u4e00\u4e2a DaemonSet \u8fd0\u884c\u5728\u6bcf\u4e2a\u8282\u70b9\u4e0a\uff0c\u5b9e\u73b0 CNI \u63a5\u53e3\uff0c\u5e76\u64cd\u4f5c\u672c\u5730\u7684 OVS \u914d\u7f6e\u5355\u673a\u7f51\u7edc\u3002</p> <p>\u8be5 DaemonSet \u4f1a\u590d\u5236 <code>kube-ovn</code> \u4e8c\u8fdb\u5236\u6587\u4ef6\u5230\u6bcf\u53f0\u673a\u5668\uff0c\u4f5c\u4e3a <code>kubelet</code> \u548c <code>kube-ovn-cni</code> \u4e4b\u95f4\u7684\u4ea4\u4e92\u5de5\u5177\uff0c\u5c06\u76f8\u5e94 CNI \u8bf7\u6c42 \u53d1\u9001\u7ed9 <code>kube-ovn-cni</code> \u6267\u884c\u3002\u8be5\u4e8c\u8fdb\u5236\u6587\u4ef6\u9ed8\u8ba4\u4f1a\u88ab\u590d\u5236\u5230 <code>/opt/cni/bin</code> \u76ee\u5f55\u4e0b\u3002</p> <p><code>kube-ovn-cni</code> \u4f1a\u914d\u7f6e\u5177\u4f53\u7684\u7f51\u7edc\u6765\u6267\u884c\u76f8\u5e94\u6d41\u91cf\u64cd\u4f5c\uff0c\u4e3b\u8981\u5de5\u4f5c\u5305\u62ec\uff1a</p> <ol> <li>\u914d\u7f6e <code>ovn-controller</code> \u548c <code>vswitchd</code>\u3002</li> <li>\u5904\u7406 CNI add/del \u8bf7\u6c42\uff1a<ol> <li>\u521b\u5efa\u5220\u9664 veth \u5e76\u548c OVS \u7aef\u53e3\u7ed1\u5b9a\u3002</li> <li>\u914d\u7f6e OVS \u7aef\u53e3\u4fe1\u606f\u3002</li> <li>\u66f4\u65b0\u5bbf\u4e3b\u673a\u7684 iptables/ipset/route \u7b49\u89c4\u5219\u3002</li> </ol> </li> <li>\u52a8\u6001\u66f4\u65b0\u5bb9\u5668 QoS.</li> <li>\u521b\u5efa\u5e76\u914d\u7f6e <code>ovn0</code> \u7f51\u5361\u8054\u901a\u5bb9\u5668\u7f51\u7edc\u548c\u4e3b\u673a\u7f51\u7edc\u3002</li> <li>\u914d\u7f6e\u4e3b\u673a\u7f51\u5361\u6765\u5b9e\u73b0 Vlan/Underlay/EIP \u7b49\u529f\u80fd\u3002</li> <li>\u52a8\u6001\u914d\u7f6e\u96c6\u7fa4\u4e92\u8054\u7f51\u5173\u3002</li> </ol>"},{"location":"reference/architecture/#_3","title":"\u76d1\u63a7\uff0c\u8fd0\u7ef4\u5de5\u5177\u548c\u6269\u5c55\u7ec4\u4ef6","text":"<p>\u8be5\u90e8\u5206\u7ec4\u4ef6\u4e3b\u8981\u63d0\u4f9b\u76d1\u63a7\uff0c\u8bca\u65ad\uff0c\u8fd0\u7ef4\u64cd\u4f5c\u4ee5\u53ca\u548c\u5916\u90e8\u8fdb\u884c\u5bf9\u63a5\uff0c\u5bf9 Kube-OVN \u7684\u6838\u5fc3\u7f51\u7edc\u80fd\u529b\u8fdb\u884c\u6269\u5c55\uff0c\u5e76\u7b80\u5316\u65e5\u5e38\u8fd0\u7ef4\u64cd\u4f5c\u3002</p>"},{"location":"reference/architecture/#kube-ovn-speaker","title":"kube-ovn-speaker","text":"<p>\u8be5\u7ec4\u4ef6\u4e3a\u4e00\u4e2a DaemonSet \u8fd0\u884c\u5728\u7279\u5b9a\u6807\u7b7e\u7684\u8282\u70b9\u4e0a\uff0c\u5bf9\u5916\u53d1\u5e03\u5bb9\u5668\u7f51\u7edc\u7684\u8def\u7531\uff0c\u4f7f\u5f97\u5916\u90e8\u53ef\u4ee5\u76f4\u63a5\u901a\u8fc7 Pod IP \u8bbf\u95ee\u5bb9\u5668\u3002</p> <p>\u66f4\u591a\u76f8\u5173\u4f7f\u7528\u65b9\u5f0f\u8bf7\u53c2\u8003 BGP \u652f\u6301\u3002</p>"},{"location":"reference/architecture/#kube-ovn-pinger","title":"kube-ovn-pinger","text":"<p>\u8be5\u7ec4\u4ef6\u4e3a\u4e00\u4e2a DaemonSet \u8fd0\u884c\u5728\u6bcf\u4e2a\u8282\u70b9\u4e0a\u6536\u96c6 OVS \u8fd0\u884c\u4fe1\u606f\uff0c\u8282\u70b9\u7f51\u7edc\u8d28\u91cf\uff0c\u7f51\u7edc\u5ef6\u8fdf\u7b49\u4fe1\u606f\uff0c\u6536\u96c6\u7684\u76d1\u63a7\u6307\u6807\u53ef\u53c2\u8003 Kube-OVN \u76d1\u63a7\u6307\u6807\u3002</p>"},{"location":"reference/architecture/#kube-ovn-monitor","title":"kube-ovn-monitor","text":"<p>\u8be5\u7ec4\u4ef6\u4e3a\u4e00\u4e2a Deployment \u6536\u96c6 OVN \u7684\u8fd0\u884c\u4fe1\u606f\uff0c\u6536\u96c6\u7684\u76d1\u63a7\u6307\u6807\u53ef\u53c2\u8003 Kube-OVN \u76d1\u63a7\u6307\u6807\u3002</p>"},{"location":"reference/architecture/#kubectl-ko","title":"kubectl-ko","text":"<p>\u8be5\u7ec4\u4ef6\u4e3a kubectl \u63d2\u4ef6\uff0c\u53ef\u4ee5\u5feb\u901f\u8fd0\u884c\u5e38\u89c1\u8fd0\u7ef4\u64cd\u4f5c\uff0c\u66f4\u591a\u4f7f\u7528\u8bf7\u53c2\u8003 kubectl \u63d2\u4ef6\u4f7f\u7528\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"reference/dev-env/","title":"\u5f00\u53d1\u73af\u5883\u6784\u5efa","text":""},{"location":"reference/dev-env/#_2","title":"\u73af\u5883\u51c6\u5907","text":"<p>Kube-OVN \u4f7f\u7528 Go 1.20 \u5f00\u53d1\u5e76\u4f7f\u7528 Go Modules \u7ba1\u7406\u4f9d\u8d56\uff0c \u8bf7\u786e\u8ba4\u73af\u5883\u53d8\u91cf <code>GO111MODULE=\"on\"</code>\u3002</p> <p>gosec \u88ab\u7528\u6765\u626b\u63cf\u4ee3\u7801\u5b89\u5168\u76f8\u5173\u95ee\u9898\uff0c\u9700\u8981\u5728\u5f00\u53d1\u73af\u5883\u5b89\u88c5\uff1a</p> <pre><code>go get github.com/securego/gosec/v2/cmd/gosec\n</code></pre> <p>\u4e3a\u4e86\u964d\u4f4e\u6700\u7ec8\u751f\u6210\u955c\u50cf\u5927\u5c0f\uff0cKube-OVN \u4f7f\u7528\u4e86\u90e8\u5206 Docker buildx \u8bd5\u9a8c\u7279\u6027\uff0c\u8bf7\u66f4\u65b0 Docker \u81f3\u6700\u65b0\u7248\u672c \u5e76\u5f00\u542f buildx:</p> <pre><code>docker buildx create --use\n</code></pre>"},{"location":"reference/dev-env/#_3","title":"\u6784\u5efa\u955c\u50cf","text":"<p>\u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u4e0b\u8f7d\u4ee3\u7801\uff0c\u5e76\u751f\u6210\u8fd0\u884c Kube-OVN \u6240\u9700\u955c\u50cf\uff1a</p> <pre><code>git clone https://github.com/kubeovn/kube-ovn.git\ncd kube-ovn\nmake release\n</code></pre> <p>\u5982\u9700\u6784\u5efa\u5728 ARM \u73af\u5883\u4e0b\u8fd0\u884c\u7684\u955c\u50cf\uff0c\u8bf7\u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\uff1a</p> <pre><code>make release-arm\n</code></pre>"},{"location":"reference/dev-env/#base","title":"\u6784\u5efa base \u955c\u50cf","text":"<p>\u5982\u9700\u8981\u66f4\u6539\u64cd\u4f5c\u7cfb\u7edf\u7248\u672c\uff0c\u4f9d\u8d56\u5e93\uff0cOVS/OVN \u4ee3\u7801\u7b49\uff0c\u9700\u8981\u5bf9 base \u955c\u50cf\u8fdb\u884c\u91cd\u65b0\u6784\u5efa\u3002</p> <p>base \u955c\u50cf\u4f7f\u7528\u7684 Dockerfile \u4e3a <code>dist/images/Dockerfile.base</code>\u3002</p> <p>\u6784\u5efa\u65b9\u6cd5\uff1a</p> <pre><code># build x86 base image\nmake base-amd64\n\n# build arm base image\nmake base-arm64\n</code></pre>"},{"location":"reference/dev-env/#e2e","title":"\u8fd0\u884c E2E","text":"<p>Kube-OVN \u4f7f\u7528 KIND \u6784\u5efa\u672c\u5730 Kubernetes \u96c6\u7fa4\uff0cj2cli \u6e32\u67d3\u6a21\u677f\uff0c Ginkgo \u6765\u8fd0\u884c\u6d4b\u8bd5\u4ee3\u7801\u3002\u8bf7\u53c2\u8003\u76f8\u5173\u6587\u6863\u8fdb\u884c\u4f9d\u8d56\u5b89\u88c5\u3002</p> <p>\u672c\u5730\u6267\u884c E2E \u6d4b\u8bd5\uff1a</p> <pre><code>make kind-init\nmake kind-install\nmake e2e\n</code></pre> <p>\u5982\u9700\u8fd0\u884c Underlay E2E \u6d4b\u8bd5\uff0c\u6267\u884c\u4e0b\u5217\u547d\u4ee4\uff1a</p> <pre><code>make kind-init\nmake kind-install-underlay\nmake e2e-underlay-single-nic\n</code></pre> <p>\u5982\u9700\u8fd0\u884c ovn vpc nat gw eip, fip, snat, dnat \u6d4b\u8bd5\uff0c\u6267\u884c\u4e0b\u5217\u547d\u4ee4\uff1a</p> <pre><code>make kind-init\nmake kind-install\nmake ovn-vpc-nat-gw-conformance-e2e\n</code></pre> <p>\u5982\u9700\u8fd0\u884c iptables vpc nat gw eip, fip, snat, dnat \u6d4b\u8bd5\uff0c\u6267\u884c\u4e0b\u5217\u547d\u4ee4\uff1a</p> <pre><code>make kind-init\nmake kind-install\nmake kind-install-vpc-nat-gw\nmake iptables-vpc-nat-gw-conformance-e2e\n</code></pre> <p>\u5982\u9700\u8fd0\u884c loadbalancer service \u6d4b\u8bd5\uff0c\u6267\u884c\u4e0b\u5217\u547d\u4ee4\uff1a</p> <pre><code>make kind-init\nmake kind-install\nmake kind-install-lb-svc\nmake kube-ovn-lb-svc-conformance-e2e\n</code></pre> <p>\u5982\u9700\u6e05\u7406\uff0c\u6267\u884c\u4e0b\u5217\u547d\u4ee4\uff1a</p> <pre><code>make kind-clean\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"reference/document-convention/","title":"\u6587\u6863\u89c4\u8303","text":"<p>\u4e3a\u4e86\u4fdd\u8bc1\u6587\u6863\u98ce\u683c\u4e00\u81f4\uff0c\u8bf7\u5728\u63d0\u4ea4\u6587\u6863\u65f6\u9075\u5faa\u4e0b\u5217\u7684\u98ce\u683c\u89c4\u8303\u3002</p>"},{"location":"reference/document-convention/#_2","title":"\u6807\u70b9","text":"<p>\u4e2d\u6587\u6587\u6863\u4e2d\u6587\u672c\u5185\u5bb9\u6240\u6709\u6807\u70b9\u5e94\u4f7f\u7528\u4e2d\u6587\u683c\u5f0f\u6807\u70b9\uff0c\u82f1\u6587\u6587\u6863\u4e2d\u6240\u6709\u6587\u672c\u5185\u5bb9\u4e2d\u5e94\u4f7f\u7528\u82f1\u6587\u6807\u70b9\u3002</p> BadGood   \u8fd9\u91cc\u63d0\u4f9b\u4e86\u4e00\u952e\u5b89\u88c5\u811a\u672c,\u53ef\u4ee5\u5e2e\u52a9\u4f60\u5feb\u901f\u5b89\u88c5\u4e00\u4e2a\u9ad8\u53ef\u7528,\u751f\u4ea7\u5c31\u7eea\u7684\u5bb9\u5668\u7f51\u7edc.    \u8fd9\u91cc\u63d0\u4f9b\u4e86\u4e00\u952e\u5b89\u88c5\u811a\u672c\uff0c\u53ef\u4ee5\u5e2e\u52a9\u4f60\u5feb\u901f\u5b89\u88c5\u4e00\u4e2a\u9ad8\u53ef\u7528\uff0c\u751f\u4ea7\u5c31\u7eea\u7684\u5bb9\u5668\u7f51\u7edc\u3002   <p>\u82f1\u6587\u6570\u5b57\u548c\u4e2d\u6587\u5e94\u8be5\u7528\u7a7a\u683c\u8fdb\u884c\u5206\u9694\u3002</p> BadGood   Kube-OVN \u63d0\u4f9b\u4e86\u4e00\u952e\u5b89\u88c5\u811a\u672c\u6765\u5b89\u88c5 1.10 \u7248\u672c Kube-OVN\u3002    Kube-OVN \u63d0\u4f9b\u4e86\u4e00\u952e\u5b89\u88c5\u811a\u672c\u6765\u5b89\u88c5 1.10 \u7248\u672c Kube-OVN\u3002   <p>\u793a\u4f8b\u5185\u5bb9\u5e94\u8be5\u4ee5 <code>\uff1a</code> \u5f00\u542f\uff0c\u5176\u4ed6\u53e5\u5c3e\u9700\u8981\u7528 <code>\u3002</code> \u7ed3\u675f\u3002</p> BadGood   \u5b89\u88c5\u524d\u8bf7\u786e\u8ba4\u73af\u5883\u914d\u7f6e\u6b63\u786e  \u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u4e0b\u8f7d\u5b89\u88c5\u811a\u672c\u3002  <pre><code>wget 127.0.0.1\n</code></pre>   \u5b89\u88c5\u524d\u8bf7\u786e\u8ba4\u73af\u5883\u914d\u7f6e\u6b63\u786e\u3002  \u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u4e0b\u8f7d\u5b89\u88c5\u811a\u672c\uff1a  <pre><code>wget 127.0.0.1\n</code></pre>"},{"location":"reference/document-convention/#_3","title":"\u4ee3\u7801\u5757","text":"<p>yaml \u4ee3\u7801\u5757\u9700\u8981\u6807\u8bc6\u4e3a yaml\u3002</p> BadGood <pre><code>````\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n    name: attach-subnet\n````\n</code></pre> <pre><code>````yaml\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n    name: attach-subnet\n````\n</code></pre> <p>\u547d\u4ee4\u884c\u64cd\u4f5c\u793a\u4f8b\u4ee3\u7801\u5757\u9700\u8981\u6807\u8bc6\u4e3a bash\u3002</p> BadGood <pre><code>````\nwget 127.0.0.1\n````\n</code></pre> <pre><code>````bash\nwget 127.0.0.1\n````\n</code></pre> <p>\u5982\u679c\u547d\u4ee4\u884c\u64cd\u4f5c\u793a\u4f8b\u4e2d\u5305\u542b\u8f93\u51fa\u5185\u5bb9\uff0c\u5219\u6240\u6267\u884c\u547d\u4ee4\u9700\u8981\u4ee5 <code>#</code> \u5f00\u59cb\uff0c\u4ee5\u533a\u5206\u8f93\u5165\u4e0e\u8f93\u51fa\u3002</p> BadGood <pre><code>oilbeater@macdeMac-3 ~ ping 114.114.114.114 -c 3\nPING 114.114.114.114 (114.114.114.114): 56 data bytes\n64 bytes from 114.114.114.114: icmp_seq=0 ttl=83 time=10.429 ms\n64 bytes from 114.114.114.114: icmp_seq=1 ttl=79 time=11.360 ms\n64 bytes from 114.114.114.114: icmp_seq=2 ttl=76 time=10.794 ms\n\n--- 114.114.114.114 ping statistics ---\n3 packets transmitted, 3 packets received, 0.0% packet loss\nround-trip min/avg/max/stddev = 10.429/10.861/11.360/0.383 ms\n</code></pre> <pre><code># ping 114.114.114.114 -c 3\nPING 114.114.114.114 (114.114.114.114): 56 data bytes\n64 bytes from 114.114.114.114: icmp_seq=0 ttl=83 time=10.429 ms\n64 bytes from 114.114.114.114: icmp_seq=1 ttl=79 time=11.360 ms\n64 bytes from 114.114.114.114: icmp_seq=2 ttl=76 time=10.794 ms\n\n--- 114.114.114.114 ping statistics ---\n3 packets transmitted, 3 packets received, 0.0% packet loss\nround-trip min/avg/max/stddev = 10.429/10.861/11.360/0.383 ms\n</code></pre> <p>\u5982\u679c\u547d\u4ee4\u884c\u64cd\u4f5c\u793a\u4f8b\u4e2d\u53ea\u5305\u542b\u6267\u884c\u547d\u4ee4\uff0c\u6ca1\u6709\u8f93\u51fa\u7ed3\u679c\uff0c\u5219\u591a\u6761\u547d\u4ee4\u65e0\u9700 <code>#</code> \u5f00\u59cb\u3002</p> BadGood <pre><code># mv /etc/origin/ovn/ovnnb_db.db /tmp\n# mv /etc/origin/ovn/ovnsb_db.db /tmp\n</code></pre> <pre><code>mv /etc/origin/ovn/ovnnb_db.db /tmp\nmv /etc/origin/ovn/ovnsb_db.db /tmp\n</code></pre>"},{"location":"reference/document-convention/#_4","title":"\u94fe\u63a5","text":"<p>\u7ad9\u5185\u94fe\u63a5\u4f7f\u7528\u5bf9\u5e94 <code>md</code> \u6587\u4ef6\u8def\u5f84\u3002</p> BadGood <pre><code>\u5b89\u88c5\u524d\u8bf7\u53c2\u8003[\u51c6\u5907\u5de5\u4f5c](http://kubeovn.github.io/prepare)\u3002\n</code></pre> <pre><code>\u5b89\u88c5\u524d\u8bf7\u53c2\u8003[\u51c6\u5907\u5de5\u4f5c](./prepare.md)\u3002\n</code></pre> BadGood <pre><code>\u5982\u6709\u95ee\u9898\u8bf7\u53c2\u8003 [Kubernetes \u6587\u6863](http://kubernetes.io)\u3002\n</code></pre> <pre><code>\u5982\u6709\u95ee\u9898\u8bf7\u53c2\u8003 [Kubernetes \u6587\u6863](http://kubernetes.io){: target=\"_blank\" }\u3002\n</code></pre>"},{"location":"reference/document-convention/#_5","title":"\u7a7a\u884c","text":"<p>\u4e0d\u540c\u903b\u8f91\u5757\uff0c\u4f8b\u5982\u6807\u9898\u548c\u6587\u672c\uff0c\u6587\u672c\u548c\u4ee3\u7801\uff0c\u6587\u672c\u548c\u7f16\u53f7\u4e4b\u95f4\u9700\u8981\u7528\u7a7a\u884c\u5206\u9694\u3002</p> BadGood <pre><code>\u4e0b\u8f7d\u4e0b\u9762\u7684\u811a\u672c\uff0c\u8fdb\u884c\u5b89\u88c5\uff1a\n```bash\nwget 127.0.0.1\n```\n</code></pre> <pre><code>\u4e0b\u8f7d\u4e0b\u9762\u7684\u811a\u672c\uff0c\u8fdb\u884c\u5b89\u88c5\uff1a\n\n```bash\nwget 127.0.0.1\n```\n</code></pre> <p>\u4e0d\u540c\u903b\u8f91\u5757\u4e4b\u95f4\u53ea\u4f7f\u7528\u4e00\u4e2a\u7a7a\u884c\u8fdb\u884c\u5206\u9694\u3002</p> BadGood <pre><code>\u4e0b\u8f7d\u4e0b\u9762\u7684\u811a\u672c\uff0c\u8fdb\u884c\u5b89\u88c5\uff1a\n\n\n\n```bash\nwget 127.0.0.1\n```\n</code></pre> <pre><code>\u4e0b\u8f7d\u4e0b\u9762\u7684\u811a\u672c\uff0c\u8fdb\u884c\u5b89\u88c5\uff1a\n\n```bash\nwget 127.0.0.1\n```\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"reference/feature-stage/","title":"\u529f\u80fd\u6210\u719f\u5ea6","text":"<p>\u5728 Kube-OVN \u4e2d\u6839\u636e\u529f\u80fd\u4f7f\u7528\u5ea6\uff0c\u6587\u6863\u5b8c\u5584\u7a0b\u5ea6\u548c\u6d4b\u8bd5\u8986\u76d6\u7a0b\u5ea6\u5c06\u529f\u80fd\u6210\u719f\u5ea6\u5206\u4e3a Alpha\uff0cBeta \u548c GA \u4e09\u4e2a\u9636\u6bb5\u3002</p>"},{"location":"reference/feature-stage/#_2","title":"\u6210\u719f\u5ea6\u5b9a\u4e49","text":"<p>\u5bf9\u4e8e Alpha \u529f\u80fd\uff1a</p> <ul> <li>\u8be5\u529f\u80fd\u6ca1\u6709\u5b8c\u6574\u7684\u6587\u6863\u548c\u5b8c\u5584\u7684\u6d4b\u8bd5\u8986\u76d6\u3002</li> <li>\u8be5\u529f\u80fd\u672a\u6765\u53ef\u80fd\u4f1a\u53d1\u751f\u53d8\u5316\u751a\u81f3\u6574\u4f53\u79fb\u9664\u3002</li> <li>\u8be5\u529f\u80fd API \u4e0d\u4fdd\u8bc1\u7a33\u5b9a\uff0c\u53ef\u80fd\u4f1a\u88ab\u79fb\u9664\u3002</li> <li>\u8be5\u529f\u80fd\u7684\u793e\u533a\u652f\u6301\u4f18\u5148\u7ea7\u8f83\u4f4e\uff0c\u4e14\u65e0\u6cd5\u4fdd\u8bc1\u957f\u671f\u652f\u6301\u3002</li> <li>\u7531\u4e8e\u529f\u80fd\u7a33\u5b9a\u6027\u548c\u957f\u671f\u652f\u6301\u65e0\u6cd5\u4fdd\u8bc1\uff0c\u53ef\u4ee5\u8fdb\u884c\u6d4b\u8bd5\u9a8c\u8bc1\uff0c\u4f46\u4e0d\u63a8\u8350\u751f\u4ea7\u4f7f\u7528\u3002</li> </ul> <p>\u5bf9\u4e8e Beta \u529f\u80fd\uff1a</p> <ul> <li>\u8be5\u529f\u80fd\u6709\u90e8\u5206\u6587\u6863\u548c\u6d4b\u8bd5\uff0c\u4f46\u662f\u4e0d\u4fdd\u8bc1\u5b8c\u6574\u7684\u8986\u76d6\u3002</li> <li>\u8be5\u529f\u80fd\u672a\u6765\u53ef\u80fd\u53d1\u751f\u53d8\u5316\uff0c\u5347\u7ea7\u53ef\u80fd\u4f1a\u5f71\u54cd\u7f51\u7edc\uff0c\u4f46\u4e0d\u4f1a\u88ab\u6574\u4f53\u79fb\u9664\u3002</li> <li>\u8be5\u529f\u80fd API \u672a\u6765\u53ef\u80fd\u4f1a\u53d1\u751f\u53d8\u5316\uff0c\u5b57\u6bb5\u53ef\u80fd\u4f1a\u8fdb\u884c\u8c03\u6574\uff0c\u4f46\u4e0d\u4f1a\u6574\u4f53\u79fb\u9664\u3002</li> <li>\u8be5\u529f\u80fd\u4f1a\u5f97\u5230\u793e\u533a\u7684\u957f\u671f\u652f\u6301\u3002</li> <li>\u7531\u4e8e\u529f\u80fd\u4f1a\u5f97\u5230\u957f\u671f\u652f\u6301\uff0c\u53ef\u4ee5\u5728\u975e\u5173\u952e\u4e1a\u52a1\u4e0a\u8fdb\u884c\u4f7f\u7528\uff0c\u4f46\u662f\u7531\u4e8e\u529f\u80fd\u548c API \u5b58\u5728\u53d8\u5316\u7684\u53ef\u80fd\uff0c\u53ef\u80fd\u4f1a\u5728\u5347\u7ea7\u4e2d\u51fa\u73b0\u4e2d\u65ad\uff0c\u4e0d\u63a8\u8350\u5728\u5173\u952e\u751f\u4ea7\u4e1a\u52a1\u4e0a\u4f7f\u7528\u3002</li> </ul> <p>\u5bf9\u4e8e GA \u529f\u80fd\uff1a</p> <ul> <li>\u8be5\u529f\u80fd\u6709\u5b8c\u6574\u7684\u6587\u6863\u548c\u6d4b\u8bd5\u8986\u76d6\u3002</li> <li>\u8be5\u529f\u80fd\u4f1a\u4fdd\u6301\u7a33\u5b9a\uff0c\u5347\u7ea7\u4f1a\u4fdd\u8bc1\u5e73\u6ed1\u3002</li> <li>\u8be5\u529f\u80fd API \u4e0d\u4f1a\u53d1\u751f\u7834\u574f\u6027\u53d8\u5316\u3002</li> <li>\u8be5\u529f\u80fd\u4f1a\u5f97\u5230\u793e\u533a\u9ad8\u4f18\u5148\u7ea7\u652f\u6301\uff0c\u5e76\u4f1a\u4fdd\u8bc1\u957f\u671f\u652f\u6301\u3002</li> </ul>"},{"location":"reference/feature-stage/#_3","title":"\u6210\u719f\u5ea6\u5217\u8868","text":"<p>\u672c\u5217\u8868\u7edf\u8ba1\u4ece v1.8 \u7248\u672c\u4e2d\u5305\u542b\u7684\u529f\u80fd\u5bf9\u5e94\u6210\u719f\u5ea6\u3002</p> \u529f\u80fd \u9ed8\u8ba4\u5f00\u542f \u72b6\u6001 \u5f00\u59cb\uff08Since\uff09 \u7ed3\u675f\uff08Until\uff09 Namespaced Subnet true GA 1.8 \u5206\u5e03\u5f0f\u7f51\u5173 true GA 1.8 \u4e3b\u4ece\u6a21\u5f0f\u96c6\u4e2d\u5f0f\u7f51\u5173 true GA 1.8 ECMP \u6a21\u5f0f\u96c6\u4e2d\u5f0f\u7f51\u5173 false Beta 1.8 \u5b50\u7f51 ACL true Alpha 1.9 \u5b50\u7f51\u9694\u79bb (\u672a\u6765\u4f1a\u548c\u5b50\u7f51 ACL \u5408\u5e76) true Beta 1.8 Underlay \u5b50\u7f51 true GA 1.8 \u591a\u7f51\u5361\u7ba1\u7406 true Beta 1.8 \u5b50\u7f51 DHCP false Alpha 1.10 \u5b50\u7f51\u8bbe\u7f6e\u5916\u90e8\u7f51\u5173 false Alpha 1.8 \u4f7f\u7528 OVN-IC \u8fdb\u884c\u96c6\u7fa4\u4e92\u8054 false Beta 1.8 \u4f7f\u7528 Submariner \u8fdb\u884c\u96c6\u7fa4\u4e92\u8054 false Alpha 1.9 \u5b50\u7f51 VIP \u9884\u7559 true Alpha 1.10 \u521b\u5efa\u81ea\u5b9a\u4e49 VPC true Beta 1.8 \u81ea\u5b9a\u4e49 VPC \u6d6e\u52a8 IP/SNAT/DNAT true Alpha 1.10 \u81ea\u5b9a\u4e49 VPC \u9759\u6001\u8def\u7531 true Alpha 1.10 \u81ea\u5b9a\u4e49 VPC \u7b56\u7565\u8def\u7531 true Alpha 1.10 \u81ea\u5b9a\u4e49 VPC \u5b89\u5168\u7ec4 true Alpha 1.10 \u5bb9\u5668\u6700\u5927\u5e26\u5bbd QoS true GA 1.8 linux-netem QoS true Alpha 1.9 Prometheus \u96c6\u6210 false GA 1.8 Grafana \u96c6\u6210 false GA 1.8 \u53cc\u6808\u7f51\u7edc false GA 1.8 \u9ed8\u8ba4 VPC EIP/SNAT false Beta 1.8 \u6d41\u91cf\u955c\u50cf false GA 1.8 NetworkPolicy true Beta 1.8 Webhook false Alpha 1.10 \u6027\u80fd\u8c03\u4f18 false Beta 1.8 Overlay \u5b50\u7f51\u9759\u6001\u8def\u7531\u5bf9\u5916\u66b4\u9732 false Alpha 1.8 Overlay \u5b50\u7f51 BGP \u5bf9\u5916\u66b4\u9732 false Alpha 1.9 Cilium \u96c6\u6210 false Alpha 1.10 \u81ea\u5b9a\u4e49 VPC \u4e92\u8054 false Alpha 1.10 Mellanox Offload false Alpha 1.8 \u82af\u542f\u6e90 Offload false Alpha 1.10 Windows \u652f\u6301 false Alpha 1.10 DPDK \u652f\u6301 false Alpha 1.10 OpenStack \u96c6\u6210 false Alpha 1.9 \u5355\u4e2a Pod \u56fa\u5b9a IP/Mac true GA 1.8 Workload \u56fa\u5b9a IP true GA 1.8 StatefulSet \u56fa\u5b9a IP true GA 1.8 VM \u56fa\u5b9a IP false Beta 1.9 \u9ed8\u8ba4 VPC Load Balancer \u7c7b\u578b Service false Alpha 1.11 \u81ea\u5b9a\u4e49 VPC \u5185\u90e8\u8d1f\u8f7d\u5747\u8861 false Alpha 1.11 \u81ea\u5b9a\u4e49 VPC DNS false Alpha 1.11 Underlay \u548c Overlay \u4e92\u901a false Alpha 1.11 <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"reference/iptables-rules/","title":"Iptables \u89c4\u5219","text":"<p>Kube-OVN \u4f7f\u7528 ipset \u53ca iptables \u8f85\u52a9\u5b9e\u73b0\u9ed8\u8ba4 VPC \u4e0b\u5bb9\u5668\u7f51\u7edc\uff08Overlay\uff09\u7f51\u5173 NAT \u7684\u529f\u80fd\u3002</p> <p>\u4f7f\u7528\u7684 ipset \u5982\u4e0b\u8868\u6240\u793a\uff1a</p> \u540d\u79f0\uff08IPv4/IPv6\uff09 \u7c7b\u578b \u5b58\u50a8\u5bf9\u8c61 ovn40services/ovn60services hash:net Service \u7f51\u6bb5 ovn40subnets/ovn60subnets hash:net Overlay \u5b50\u7f51\u7f51\u6bb5\u4ee5\u53ca NodeLocal DNS IP \u5730\u5740 ovn40subnets-nat/ovn60subnets-nat hash:net \u5f00\u542f NatOutgoing \u7684 Overlay \u5b50\u7f51\u7f51\u6bb5 ovn40subnets-distributed-gw/ovn60subnets-distributed-gw hash:net \u5f00\u542f\u5206\u5e03\u5f0f\u7f51\u5173\u7684 Overlay \u5b50\u7f51\u7f51\u6bb5 ovn40other-node/ovn60other-node hash:net \u5176\u5b83\u8282\u70b9\u7684\u5185\u90e8 IP \u5730\u5740 ovn40local-pod-ip-nat/ovn60local-pod-ip-nat hash:ip \u5df2\u5f03\u7528 ovn40subnets-nat-policy hash:net \u914d\u7f6e\u4e86 natOutgoingPolicyRules \u7684\u6240\u6709\u5b50\u7f51\u7f51\u6bb5 ovn40natpr-418e79269dc5-dst hash:net natOutgoingPolicyRules \u4e2d rule \u5bf9\u5e94\u7684 dstIPs ovn40natpr-418e79269dc5-src hash:net natOutgoingPolicyRules \u4e2d rule \u5bf9\u5e94\u7684 srcIPs <p>\u4f7f\u7528\u7684 iptables \u89c4\u5219\uff08IPv4\uff09\u5982\u4e0b\u8868\u6240\u793a\uff1a</p> \u8868 \u94fe \u89c4\u5219 \u7528\u9014 \u5907\u6ce8 filter INPUT -m set --match-set ovn40services src -j ACCEPT \u5141\u8bb8 k8s Service \u548c Pod \u76f8\u5173\u6d41\u91cf\u901a\u8fc7 -- filter INPUT -m set --match-set ovn40services dst -j ACCEPT \u540c\u4e0a -- filter INPUT -m set --match-set ovn40subnets src -j ACCEPT \u540c\u4e0a -- filter INPUT -m set --match-set ovn40subnets dst -j ACCEPT \u540c\u4e0a -- filter FORWARD -m set --match-set ovn40services src -j ACCEPT \u540c\u4e0a -- filter FORWARD -m set --match-set ovn40services dst -j ACCEPT \u540c\u4e0a -- filter FORWARD -m set --match-set ovn40subnets src -j ACCEPT \u540c\u4e0a -- filter FORWARD -m set --match-set ovn40subnets dst -j ACCEPT \u540c\u4e0a -- filter FORWARD -s 10.16.0.0/16 -m comment --comment \"ovn-subnet-gateway,ovn-default\" \u7528\u4e8e\u8ba1\u6570\u4ece subnet \u8bbf\u95ee\u5916\u90e8\u7f51\u7edc\u7684\u62a5\u6587 10.16.0.0/16 \u4e3a subnet \u7684 cidr \uff0ccomment \u4e2d\u9017\u53f7\u524d\u9762\u7684 ovn-subnet-gateway \u7528\u4e8e\u6807\u8bc6\u8be5 iptables \u89c4\u5219\u7528\u4e8e subnet \u51fa\u5165\u7f51\u5173\u62a5\u6587\u8ba1\u6570\uff0c\u9017\u53f7\u540e\u9762 ovn-default \u662f\u8be5 subnet \u7684\u540d\u5b57 filter FORWARD -d 10.16.0.0/16 -m comment --comment \"ovn-subnet-gateway,ovn-default\" \u7528\u4e8e\u8ba1\u6570\u4ece\u5916\u90e8\u7f51\u7edc\u8bbf\u95ee subnet \u7684\u62a5\u6587 \u540c\u4e0a filter OUTPUT -p udp -m udp --dport 6081 -j MARK --set-xmark 0x0 \u6e05\u9664\u6d41\u91cf\u6807\u8bb0\uff0c\u907f\u514d\u6267\u884c SNAT UDP: bad checksum on VXLAN interface nat PREROUTING -m comment --comment \"kube-ovn prerouting rules\" -j OVN-PREROUTING \u8fdb\u5165 OVN-PREROUTING \u94fe\u5904\u7406 -- nat POSTROUTING -m comment --comment \"kube-ovn postrouting rules\" -j OVN-POSTROUTING \u8fdb\u5165 OVN-POSTROUTING \u94fe\u5904\u7406 -- nat OVN-PREROUTING -i ovn0 -m set --match-set ovn40subnets src -m set --match-set ovn40services dst -j MARK --set-xmark 0x4000/0x4000 \u4e3a Pod \u8bbf\u95ee Service \u6d41\u91cf\u6dfb\u52a0 masquerade \u6807\u8bb0 \u4f5c\u7528\u4e8e\u5173\u95ed\u5185\u7f6e LB \u7684\u573a\u666f nat OVN-PREROUTING -p tcp -m addrtype --dst-type LOCAL -m set --match-set KUBE-NODE-PORT-LOCAL-TCP dst -j MARK --set-xmark 0x80000/0x80000 \u4e3a ExternalTrafficPolicy \u4e3a Local \u7684 Service \u6d41\u91cf\uff08TCP\uff09\u6dfb\u52a0\u7279\u5b9a\u6807\u8bb0 \u4ec5 kube-proxy \u4f7f\u7528 ipvs \u6a21\u5f0f\u65f6\u5b58\u5728 nat OVN-PREROUTING -p udp -m addrtype --dst-type LOCAL -m set --match-set KUBE-NODE-PORT-LOCAL-UDP dst -j MARK --set-xmark 0x80000/0x80000 \u4e3a ExternalTrafficPolicy \u4e3a Local \u7684 Service \u6d41\u91cf\uff08UDP\uff09\u6dfb\u52a0\u7279\u5b9a\u6807\u8bb0 \u540c\u4e0a nat OVN-POSTROUTING -m set --match-set ovn40services src -m set --match-set ovn40subnets dst -m mark --mark 0x4000/0x4000 -j SNAT --to-source  \u5f53\u8282\u70b9\u901a\u8fc7 Service IP \u8bbf\u95ee Overlay Pod \u65f6\uff0c\u4fdd\u6301\u6e90 IP \u4e3a\u8282\u70b9 IP\u3002 \u4ec5 kube-proxy \u4f7f\u7528 ipvs \u6a21\u5f0f\u65f6\u751f\u6548 nat OVN-POSTROUTING -m mark --mark 0x4000/0x4000 -j MASQUERADE \u4e3a\u7279\u5b9a\u6807\u8bb0\u7684\u6d41\u91cf\u6267\u884c SNAT -- nat OVN-POSTROUTING -m set --match-set ovn40subnets src -m set --match-set ovn40subnets dst -j MASQUERADE \u4e3a\u901a\u8fc7\u8282\u70b9\u7684 Pod \u4e4b\u95f4\u7684 Service \u6d41\u91cf\u6267\u884c SNAT -- nat OVN-POSTROUTING -m mark --mark 0x80000/0x80000 -m set --match-set ovn40subnets-distributed-gw dst -j RETURN \u5bf9\u4e8e ExternalTrafficPolicy \u4e3a Local \u7684 Service \u6d41\u91cf\uff0c\u82e5 Endpoint \u4f7f\u7528\u5206\u5e03\u5f0f\u7f51\u5173\uff0c\u65e0\u9700\u6267\u884c SNAT -- nat OVN-POSTROUTING -m mark --mark 0x80000/0x80000 -j MASQUERADE \u5bf9\u4e8e ExternalTrafficPolicy \u4e3a Local \u7684 Service \u6d41\u91cf\uff0c\u82e5 Endpoint \u4f7f\u7528\u96c6\u4e2d\u5f0f\u7f51\u5173\uff0c\u6267\u884c SNAT -- nat OVN-POSTROUTING -p tcp -m tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN Pod IP \u5bf9\u5916\u66b4\u9732\u65f6\uff0c\u4e0d\u6267\u884c SNAT -- nat OVN-POSTROUTING -s 10.16.0.0/16 -m set ! --match-set ovn40subnets dst -j SNAT --to-source 192.168.0.101 Pod \u8bbf\u95ee\u96c6\u7fa4\u5916\u7f51\u7edc\u65f6\uff0c\u82e5\u5b50\u7f51\u5f00\u542f NatOutgoing \u4e14\u4f7f\u7528\u6307\u5b9a IP \u7684\u96c6\u4e2d\u5f0f\u7f51\u5173\uff0c\u6267\u884c SNAT 10.16.0.0/16 \u4e3a\u5b50\u7f51\u7f51\u6bb5\uff0c192.168.0.101 \u4e3a\u6307\u5b9a\u7684\u7f51\u5173\u8282\u70b9 IP nat OVN-POSTROUTING -m set --match-set ovn40subnets-nat src -m set ! --match-set ovn40subnets dst -j MASQUERADE Pod \u8bbf\u95ee\u96c6\u7fa4\u5916\u7f51\u7edc\u65f6\uff0c\u82e5\u5b50\u7f51\u5f00\u542f NatOutgoing\uff0c\u6267\u884c SNAT -- nat OVN-POSTROUTING -m set --match-set ovn40subnets-nat-policy src -m set ! --match-set ovn40subnets dst -j OVN-NAT-POLICY Pod \u8bbf\u95ee\u96c6\u7fa4\u5916\u7f51\u7edc\u65f6\uff0c\u82e5\u5b50\u7f51\u5f00\u542f natOutgoingPolicyRules\uff0c\u6307\u5b9a\u7b56\u7565\u7684\u62a5\u6587\u6267\u884c SNAT \u914d\u7f6e\u4e86 natOutgoingPolicyRules \u5b50\u7f51\u7684\u51fa\u5916\u7f51\u62a5\u6587\u7684\u8fdb\u5165\u94fe OVN-NAT-POLICY nat OVN-POSTROUTING -m mark --mark 0x90001/0x90001 -j MASQUERADE --random-fully \u540c\u4e0a \u4ece OVN-NAT-POLICY \u51fa\u6765\u540e\uff0c\u5982\u679c\u88ab\u6253\u4e0a tag 0x90001/0x90001 \u5c31\u4f1a\u505a SNAT nat OVN-POSTROUTING -m mark --mark 0x90002/0x90002 -j RETURN \u540c\u4e0a \u4ece OVN-NAT-POLICY \u51fa\u6765\u540e, \u5982\u679c\u88ab\u6253\u4e0a tag 0x90002/0x90002 \u4e0d\u4f1a\u505a SNAT nat OVN-NAT-POLICY -s 10.0.11.0/24 -m comment --comment natPolicySubnet-net1 -j OVN-NAT-PSUBNET-aa98851157c5 \u540c\u4e0a 10.0.11.0/24 \u8868\u793a\u5b50\u7f51 net1 \u7684 CIDR\uff0c  OVN-NAT-PSUBNET-aa98851157c5 \u8fd9\u6761\u94fe\u4e0b\u7684\u89c4\u5219\u5c31\u5bf9\u5e94\u8fd9\u4e2a\u5b50\u7f51\u7684 natOutgoingPolicyRules \u914d\u7f6e nat OVN-NAT-PSUBNET-xxxxxxxxxxxx -m set --match-set ovn40natpr-418e79269dc5-src src -m set --match-set ovn40natpr-418e79269dc5-dst dst -j MARK --set-xmark 0x90002/0x90002 \u540c\u4e0a 418e79269dc5 \u8868\u793a natOutgoingPolicyRules \u4e2d\u7684\u4e00\u6761\u89c4\u5219\u7684 ID\uff0c\u53ef\u4ee5\u901a\u8fc7 status.natOutgoingPolicyRules[index].RuleID \u67e5\u770b\u5230\uff0c \u8868\u793a srcIPs \u6ee1\u8db3 ovn40natpr-418e79269dc5-src\uff0c dstIPS \u6ee1\u8db3 ovn40natpr-418e79269dc5-dst \u4f1a\u6253\u4e0a tag 0x90002 mangle OVN-OUTPUT -d 10.241.39.2/32 -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x90003/0x90003 \u5c06 kubelet \u7684\u63a2\u6d4b\u6d41\u91cf\u52a0\u4e0a\u7279\u5b9a\u6807\u8bb0\u4ece\u800c\u5f15\u5165\u5230 tproxy mangle OVN-PREROUTING -d 10.241.39.2/32 -p tcp -m tcp --dport 80 -j TPROXY --on-port 8102 --on-ip 172.18.0.3 --tproxy-mark 0x90004/0x90004 \u5c06 kubelet \u7684\u63a2\u6d4b\u6d41\u91cf\u52a0\u4e0a\u7279\u5b9a\u6807\u8bb0\u4ece\u800c\u5f15\u5165\u5230 tproxy <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"reference/kube-ovn-api/","title":"Kube-OVN \u63a5\u53e3\u89c4\u8303","text":"<p>\u57fa\u4e8e Kube-OVN v1.12.0 \u7248\u672c\uff0c\u6574\u7406\u4e86 Kube-OVN \u652f\u6301\u7684 CRD \u8d44\u6e90\u5217\u8868\uff0c\u5217\u51fa CRD \u5b9a\u4e49\u5404\u5b57\u6bb5\u7684\u53d6\u503c\u7c7b\u578b\u548c\u542b\u4e49\uff0c\u4ee5\u4f9b\u53c2\u8003\u3002</p>"},{"location":"reference/kube-ovn-api/#condition","title":"\u901a\u7528\u7684 Condition \u5b9a\u4e49","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 type String \u72b6\u6001\u7c7b\u578b status String \u72b6\u6001\u503c\uff0c\u53d6\u503c\u4e3a <code>True</code>\uff0c<code>False</code> \u6216 <code>Unknown</code> reason String \u72b6\u6001\u53d8\u5316\u7684\u539f\u56e0 message String \u72b6\u6001\u53d8\u5316\u7684\u5177\u4f53\u4fe1\u606f lastUpdateTime Time \u4e0a\u6b21\u72b6\u6001\u66f4\u65b0\u65f6\u95f4 lastTransitionTime Time \u4e0a\u6b21\u72b6\u6001\u7c7b\u578b\u53d1\u751f\u53d8\u5316\u7684\u65f6\u95f4 <p>\u5728\u5404 CRD \u7684\u5b9a\u4e49\u4e2d\uff0cStatus \u4e2d\u7684 Condition \u5b57\u6bb5\uff0c\u90fd\u9075\u5faa\u4e0a\u8ff0\u683c\u5f0f\uff0c\u56e0\u6b64\u63d0\u524d\u8fdb\u884c\u8bf4\u660e\u3002</p>"},{"location":"reference/kube-ovn-api/#subnet","title":"Subnet \u5b9a\u4e49","text":""},{"location":"reference/kube-ovn-api/#subnet_1","title":"Subnet","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>Subnet</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec SubnetSpec Subnet \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status SubnetStatus Subnet \u72b6\u6001\u4fe1\u606f\u5b57\u6bb5"},{"location":"reference/kube-ovn-api/#subnetspec","title":"SubnetSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 default Bool \u8be5\u5b50\u7f51\u662f\u5426\u4e3a\u9ed8\u8ba4\u5b50\u7f51 vpc String \u5b50\u7f51\u6240\u5c5e Vpc\uff0c\u9ed8\u8ba4\u4e3a ovn-cluster protocol String IP \u534f\u8bae\uff0c\u53d6\u503c\u53ef\u4ee5\u4e3a <code>IPv4</code>\uff0c<code>IPv6</code> \u6216 <code>Dual</code> namespaces []String \u8be5\u5b50\u7f51\u6240\u7ed1\u5b9a\u7684 namespace \u5217\u8868 cidrBlock String \u5b50\u7f51\u7684\u7f51\u6bb5\u8303\u56f4\uff0c\u5982 10.16.0.0/16 gateway String \u5b50\u7f51\u7f51\u5173\u5730\u5740\uff0c\u9ed8\u8ba4\u4e3a\u8be5\u5b50\u7f51 CIDRBlock \u4e0b\u7684\u7b2c\u4e00\u4e2a\u53ef\u7528\u5730\u5740 excludeIps []String \u8be5\u5b50\u7f51\u4e0b\u4e0d\u4f1a\u88ab\u81ea\u52a8\u5206\u914d\u7684\u5730\u5740\u8303\u56f4 provider String \u9ed8\u8ba4\u4e3a ovn\u3002\u591a\u7f51\u5361\u60c5\u51b5\u4e0b\u53ef\u4ee5\u914d\u7f6e\u53d6\u503c\u4e3a NetworkAttachmentDefinition \u7684 .\uff0cKube-OVN \u5c06\u4f1a\u4f7f\u7528\u8fd9\u4e9b\u4fe1\u606f\u627e\u5230\u5bf9\u5e94\u7684 Subnet \u8d44\u6e90 gatewayType String Overlay \u6a21\u5f0f\u4e0b\u7684\u7f51\u5173\u7c7b\u578b\uff0c\u53d6\u503c\u53ef\u4ee5\u4e3a <code>distributed</code> \u6216 <code>centralized</code> gatewayNode String \u5f53\u7f51\u5173\u6a21\u5f0f\u4e3a centralized \u65f6\u7684\u7f51\u5173\u8282\u70b9\uff0c\u53ef\u4ee5\u4e3a\u9017\u53f7\u5206\u9694\u7684\u591a\u4e2a\u8282\u70b9 natOutgoing Bool \u51fa\u7f51\u6d41\u91cf\u662f\u5426\u8fdb\u884c NAT\u3002\u8be5\u53c2\u6570\u548c <code>externalEgressGateway</code> \u53c2\u6570\u4e0d\u80fd\u540c\u65f6\u8bbe\u7f6e\u3002 externalEgressGateway String \u5916\u90e8\u7f51\u5173\u5730\u5740\u3002\u9700\u8981\u548c\u5b50\u7f51\u7f51\u5173\u8282\u70b9\u5728\u540c\u4e00\u4e2a\u4e8c\u5c42\u53ef\u8fbe\u57df\uff0c\u8be5\u53c2\u6570\u548c <code>natOutgoing</code> \u53c2\u6570\u4e0d\u80fd\u540c\u65f6\u8bbe\u7f6e policyRoutingPriority Uint32 \u7b56\u7565\u8def\u7531\u4f18\u5148\u7ea7\u3002\u6dfb\u52a0\u7b56\u7565\u8def\u7531\u4f7f\u7528\u53c2\u6570\uff0c\u63a7\u5236\u6d41\u91cf\u7ecf\u5b50\u7f51\u7f51\u5173\u4e4b\u540e\uff0c\u8f6c\u53d1\u5230\u5916\u90e8\u7f51\u5173\u5730\u5740 policyRoutingTableID Uint32 \u4f7f\u7528\u7684\u672c\u5730\u7b56\u7565\u8def\u7531\u8868\u7684 TableID\uff0c\u6bcf\u4e2a\u5b50\u7f51\u5747\u9700\u4e0d\u540c\u4ee5\u907f\u514d\u51b2\u7a81 private Bool \u6807\u8bc6\u8be5\u5b50\u7f51\u662f\u5426\u4e3a\u79c1\u6709\u5b50\u7f51\uff0c\u79c1\u6709\u5b50\u7f51\u9ed8\u8ba4\u62d2\u7edd\u5b50\u7f51\u5916\u7684\u5730\u5740\u8bbf\u95ee allowSubnets []String \u5b50\u7f51\u4e3a\u79c1\u6709\u5b50\u7f51\u7684\u60c5\u51b5\u4e0b\uff0c\u5141\u8bb8\u8bbf\u95ee\u8be5\u5b50\u7f51\u5730\u5740\u7684\u96c6\u5408 vlan String \u5b50\u7f51\u7ed1\u5b9a\u7684 Vlan \u540d\u79f0 vips []String \u5b50\u7f51\u4e0b virtual \u7c7b\u578b lsp \u7684 virtual-ip \u53c2\u6570\u4fe1\u606f logicalGateway Bool \u662f\u5426\u542f\u7528\u903b\u8f91\u7f51\u5173 disableGatewayCheck Bool \u521b\u5efa Pod \u65f6\u662f\u5426\u8df3\u8fc7\u7f51\u5173\u8054\u901a\u6027\u68c0\u67e5 disableInterConnection Bool \u63a7\u5236\u662f\u5426\u5f00\u542f\u5b50\u7f51\u8de8\u96c6\u7fa4\u4e92\u8054 enableDHCP Bool \u63a7\u5236\u662f\u5426\u914d\u7f6e\u5b50\u7f51\u4e0b lsp \u7684 dhcp \u914d\u7f6e\u9009\u9879 dhcpV4Options String \u5b50\u7f51\u4e0b lsp dhcpv4_options \u5173\u8054\u7684 DHCP_Options \u8bb0\u5f55 dhcpV6Options String \u5b50\u7f51\u4e0b lsp dhcpv6_options \u5173\u8054\u7684 DHCP_Options \u8bb0\u5f55 enableIPv6RA Bool \u63a7\u5236\u5b50\u7f51\u8fde\u63a5\u8def\u7531\u5668\u7684 lrp \u7aef\u53e3\uff0c\u662f\u5426\u914d\u7f6e ipv6_ra_configs \u53c2\u6570 ipv6RAConfigs String \u5b50\u7f51\u8fde\u63a5\u8def\u7531\u5668\u7684 lrp \u7aef\u53e3\uff0cipv6_ra_configs \u53c2\u6570\u914d\u7f6e\u4fe1\u606f acls []Acl \u5b50\u7f51\u5bf9\u5e94 logical-switch \u5173\u8054\u7684 acls \u8bb0\u5f55 u2oInterconnection Bool \u662f\u5426\u5f00\u542f Overlay/Underlay \u7684\u4e92\u8054\u6a21\u5f0f enableLb *Bool \u63a7\u5236\u5b50\u7f51\u5bf9\u5e94\u7684 logical-switch \u662f\u5426\u5173\u8054 load-balancer \u8bb0\u5f55 enableEcmp Bool \u96c6\u4e2d\u5f0f\u7f51\u5173\uff0c\u662f\u5426\u5f00\u542f ECMP \u8def\u7531"},{"location":"reference/kube-ovn-api/#acl","title":"Acl","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 direction String Acl \u9650\u5236\u65b9\u5411\uff0c\u53d6\u503c\u4e3a <code>from-lport</code> \u6216\u8005 <code>to-lport</code> priority Int Acl \u4f18\u5148\u7ea7\uff0c\u53d6\u503c\u8303\u56f4 0 \u5230 32767 match String Acl \u89c4\u5219\u5339\u914d\u8868\u8fbe\u5f0f action String Acl \u89c4\u5219\u52a8\u4f5c\uff0c\u53d6\u503c\u4e3a <code>allow-related</code>, <code>allow-stateless</code>, <code>allow</code>, <code>drop</code>, <code>reject</code> \u5176\u4e2d\u4e00\u4e2a"},{"location":"reference/kube-ovn-api/#subnetstatus","title":"SubnetStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 conditions []SubnetCondition \u5b50\u7f51\u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49 v4AvailableIPs Float64 \u5b50\u7f51\u73b0\u5728\u53ef\u7528\u7684 IPv4 IP \u5730\u5740\u6570\u91cf v4availableIPrange String \u5b50\u7f51\u73b0\u5728\u53ef\u7528\u7684 IPv4 \u5730\u5740\u8303\u56f4 v4UsingIPs Float64 \u5b50\u7f51\u73b0\u5728\u5df2\u7528\u7684 IPv4 IP \u5730\u5740\u6570\u91cf v4usingIPrange String \u5b50\u7f51\u73b0\u5728\u5df2\u7528\u7684 IPv4 \u5730\u5740\u8303\u56f4 v6AvailableIPs Float64 \u5b50\u7f51\u73b0\u5728\u53ef\u7528\u7684 IPv6 IP \u5730\u5740\u6570\u91cf v6availableIPrange String \u5b50\u7f51\u73b0\u5728\u53ef\u7528\u7684 IPv6 \u5730\u5740\u8303\u56f4 v6UsingIPs Float64 \u5b50\u7f51\u73b0\u5728\u5df2\u7528\u7684 IPv6 IP \u5730\u5740\u6570\u91cf v6usingIPrange String \u5b50\u7f51\u73b0\u5728\u5df2\u7528\u7684 IPv6 \u5730\u5740\u8303\u56f4 sctivateGateway String \u96c6\u4e2d\u5f0f\u5b50\u7f51\uff0c\u4e3b\u5907\u6a21\u5f0f\u4e0b\u5f53\u524d\u6b63\u5728\u5de5\u4f5c\u7684\u7f51\u5173\u8282\u70b9 dhcpV4OptionsUUID String \u5b50\u7f51\u4e0b lsp dhcpv4_options \u5173\u8054\u7684 DHCP_Options \u8bb0\u5f55\u6807\u8bc6 dhcpV6OptionsUUID String \u5b50\u7f51\u4e0b lsp dhcpv6_options \u5173\u8054\u7684 DHCP_Options \u8bb0\u5f55\u6807\u8bc6 u2oInterconnectionIP String \u5f00\u542f Overlay/Underlay \u4e92\u8054\u6a21\u5f0f\u540e\uff0c\u6240\u5360\u7528\u7684\u7528\u4e8e\u4e92\u8054\u7684 IP \u5730\u5740"},{"location":"reference/kube-ovn-api/#ip","title":"IP \u5b9a\u4e49","text":""},{"location":"reference/kube-ovn-api/#ip_1","title":"IP","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>IP</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec IPSpec IP \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5"},{"location":"reference/kube-ovn-api/#ipsepc","title":"IPSepc","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 podName String \u7ed1\u5b9a Pod \u540d\u79f0 namespace String \u7ed1\u5b9a Pod \u6240\u5728 Namespace \u540d\u79f0 subnet String IP \u6240\u5c5e Subnet attachSubnets []String \u8be5\u4e3b IP \u4e0b\u5176\u4ed6\u9644\u5c5e\u5b50\u7f51\u540d\u79f0\uff08\u5b57\u6bb5\u5e9f\u5f03\u4e0d\u518d\u4f7f\u7528\uff09 nodeName String \u7ed1\u5b9a Pod \u6240\u5728\u7684\u8282\u70b9\u540d\u79f0 ipAddress String IP \u5730\u5740\uff0c\u53cc\u6808\u60c5\u51b5\u4e0b\u4e3a <code>v4IP,v6IP</code> \u683c\u5f0f v4IPAddress String IPv4 IP \u5730\u5740 v6IPAddress String IPv6 IP \u5730\u5740 attachIPs []String \u8be5\u4e3b IP \u4e0b\u5176\u4ed6\u9644\u5c5e IP \u5730\u5740\uff08\u5b57\u6bb5\u5e9f\u5f03\u4e0d\u518d\u4f7f\u7528\uff09 macAddress String \u7ed1\u5b9a Pod \u7684 Mac \u5730\u5740 attachMacs []String \u8be5\u4e3b IP \u4e0b\u5176\u4ed6\u9644\u5c5e Mac \u5730\u5740\uff08\u5b57\u6bb5\u5e9f\u5f03\u4e0d\u518d\u4f7f\u7528\uff09 containerID String \u7ed1\u5b9a Pod \u5bf9\u5e94\u7684 Container ID podType String \u7279\u6b8a\u5de5\u4f5c\u8d1f\u8f7d Pod\uff0c\u53ef\u4e3a <code>StatefulSet</code>\uff0c<code>VirtualMachine</code> \u6216\u7a7a"},{"location":"reference/kube-ovn-api/#underlay","title":"Underlay \u914d\u7f6e","text":""},{"location":"reference/kube-ovn-api/#vlan","title":"Vlan","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>Vlan</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec VlanSpec Vlan \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status VlanStatus Vlan \u72b6\u6001\u4fe1\u606f\u5b57\u6bb5"},{"location":"reference/kube-ovn-api/#vlanspec","title":"VlanSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 id Int Vlan tag \u53f7\uff0c\u53d6\u503c\u8303\u56f4\u4e3a 0~4096 provider String Vlan \u7ed1\u5b9a\u7684 ProviderNetwork \u540d\u79f0"},{"location":"reference/kube-ovn-api/#vlanstatus","title":"VlanStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 subnets []String Vlan \u7ed1\u5b9a\u7684\u5b50\u7f51\u5217\u8868 conditions []VlanCondition Vlan \u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49"},{"location":"reference/kube-ovn-api/#providernetwork","title":"ProviderNetwork","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>ProviderNetwork</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec ProviderNetworkSpec ProviderNetwork \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status ProviderNetworkStatus ProviderNetwork \u72b6\u6001\u4fe1\u606f\u5b57\u6bb5"},{"location":"reference/kube-ovn-api/#providernetworkspec","title":"ProviderNetworkSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 defaultInterface String \u8be5\u6865\u63a5\u7f51\u7edc\u9ed8\u8ba4\u4f7f\u7528\u7684\u7f51\u5361\u63a5\u53e3\u540d\u79f0 customInterfaces []CustomInterface \u8be5\u6865\u63a5\u7f51\u7edc\u7279\u6b8a\u4f7f\u7528\u7684\u7f51\u5361\u914d\u7f6e excludeNodes []String \u8be5\u6865\u63a5\u7f51\u7edc\u4e0d\u4f1a\u7ed1\u5b9a\u7684\u8282\u70b9\u540d\u79f0 exchangeLinkName Bool \u662f\u5426\u4ea4\u6362\u6865\u63a5\u7f51\u5361\u548c\u5bf9\u5e94 OVS \u7f51\u6865\u540d\u79f0"},{"location":"reference/kube-ovn-api/#custominterface","title":"CustomInterface","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 interface String Underlay \u4f7f\u7528\u7f51\u5361\u63a5\u53e3\u540d\u79f0 nodes []String \u4f7f\u7528\u81ea\u5b9a\u4e49\u7f51\u5361\u63a5\u53e3\u7684\u8282\u70b9\u5217\u8868"},{"location":"reference/kube-ovn-api/#providernetworkstatus","title":"ProviderNetworkStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 ready Bool \u5f53\u524d\u6865\u63a5\u7f51\u7edc\u662f\u5426\u8fdb\u5165\u5c31\u7eea\u72b6\u6001 readyNodes []String \u6865\u63a5\u7f51\u7edc\u8fdb\u5165\u5c31\u7eea\u72b6\u6001\u7684\u8282\u70b9\u540d\u79f0 notReadyNodes []String \u6865\u63a5\u7f51\u7edc\u672a\u8fdb\u5165\u5c31\u7eea\u72b6\u6001\u7684\u8282\u70b9\u540d\u79f0 vlans []String \u6865\u63a5\u7f51\u7edc\u7ed1\u5b9a\u7684 Vlan \u540d\u79f0 conditions []ProviderNetworkCondition ProviderNetwork \u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49"},{"location":"reference/kube-ovn-api/#vpc","title":"Vpc \u5b9a\u4e49","text":""},{"location":"reference/kube-ovn-api/#vpc_1","title":"Vpc","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>Vpc</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec VpcSpec Vpc \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status VpcStatus Vpc \u72b6\u6001\u4fe1\u606f\u5b57\u6bb5"},{"location":"reference/kube-ovn-api/#vpcspec","title":"VpcSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 namespaces []String Vpc \u7ed1\u5b9a\u7684\u547d\u540d\u7a7a\u95f4\u5217\u8868 staticRoutes []*StaticRoute Vpc \u4e0b\u914d\u7f6e\u7684\u9759\u6001\u8def\u7531\u4fe1\u606f policyRoutes []*PolicyRoute Vpc \u4e0b\u914d\u7f6e\u7684\u7b56\u7565\u8def\u7531\u4fe1\u606f vpcPeerings []*VpcPeering Vpc \u4e92\u8054\u4fe1\u606f enableExternal Bool Vpc \u662f\u5426\u8fde\u63a5\u5230\u5916\u90e8\u4ea4\u6362\u673a"},{"location":"reference/kube-ovn-api/#staticroute","title":"StaticRoute","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 policy String \u8def\u7531\u7b56\u7565\uff0c\u53d6\u503c\u4e3a <code>policySrc</code> \u6216\u8005 <code>policyDst</code> cidr String \u8def\u7531 Cidr \u7f51\u6bb5 nextHopIP String \u8def\u7531\u4e0b\u4e00\u8df3\u4fe1\u606f"},{"location":"reference/kube-ovn-api/#policyroute","title":"PolicyRoute","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 priority Int32 \u7b56\u7565\u8def\u7531\u4f18\u5148\u7ea7 match String \u7b56\u7565\u8def\u7531\u5339\u914d\u6761\u4ef6 action String \u7b56\u7565\u8def\u7531\u52a8\u4f5c\uff0c\u53d6\u503c\u4e3a <code>allow</code>\u3001<code>drop</code> \u6216\u8005 <code>reroute</code> nextHopIP String \u7b56\u7565\u8def\u7531\u4e0b\u4e00\u8df3\u4fe1\u606f\uff0cECMP \u8def\u7531\u60c5\u51b5\u4e0b\u4e0b\u4e00\u8df3\u5730\u5740\u4f7f\u7528\u9017\u53f7\u9694\u5f00"},{"location":"reference/kube-ovn-api/#vpcpeering","title":"VpcPeering","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 remoteVpc String Vpc \u4e92\u8054\u5bf9\u7aef Vpc \u540d\u79f0 localConnectIP String Vpc \u4e92\u8054\u672c\u7aef IP \u5730\u5740"},{"location":"reference/kube-ovn-api/#vpcstatus","title":"VpcStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 conditions []VpcCondition Vpc \u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49 standby Bool \u6807\u8bc6 Vpc \u662f\u5426\u521b\u5efa\u5b8c\u6210\uff0cVpc \u4e0b\u7684 Subnet \u9700\u8981\u7b49 Vpc \u521b\u5efa\u5b8c\u6210\u8f6c\u6362\u518d\u7ee7\u7eed\u5904\u7406 default Bool \u662f\u5426\u662f\u9ed8\u8ba4 Vpc defaultLogicalSwitch String Vpc \u4e0b\u7684\u9ed8\u8ba4\u5b50\u7f51 router String Vpc \u5bf9\u5e94\u7684 logical-router \u540d\u79f0 tcpLoadBalancer String Vpc \u4e0b\u7684 TCP LB \u4fe1\u606f udpLoadBalancer String Vpc \u4e0b\u7684 UDP LB \u4fe1\u606f tcpSessionLoadBalancer String Vpc \u4e0b\u7684 TCP \u4f1a\u8bdd\u4fdd\u6301 LB \u4fe1\u606f udpSessionLoadBalancer String Vpc \u4e0b\u7684 UDP \u4f1a\u8bdd\u4fdd\u6301 LB \u4fe1\u606f subnets []String Vpc \u4e0b\u7684\u5b50\u7f51\u5217\u8868 vpcPeerings []String Vpc \u4e92\u8054\u7684\u5bf9\u7aef Vpc \u5217\u8868 enableExternal Bool Vpc \u662f\u5426\u8fde\u63a5\u5230\u5916\u90e8\u4ea4\u6362\u673a"},{"location":"reference/kube-ovn-api/#vpcnatgateway","title":"VpcNatGateway","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>VpcNatGateway</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec VpcNatSpec Vpc \u7f51\u5173\u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5"},{"location":"reference/kube-ovn-api/#vpcnatspec","title":"VpcNatSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 vpc String Vpc \u7f51\u5173 Pod \u6240\u5728\u7684 Vpc \u540d\u79f0 subnet String Vpc \u7f51\u5173 Pod \u6240\u5c5e\u7684\u5b50\u7f51\u540d\u79f0 lanIp String Vpc \u7f51\u5173 Pod \u6307\u5b9a\u5206\u914d\u7684 IP \u5730\u5740 selector []String \u6807\u51c6 Kubernetes Selector \u5339\u914d\u4fe1\u606f tolerations []VpcNatToleration \u6807\u51c6 Kubernetes \u5bb9\u5fcd\u4fe1\u606f"},{"location":"reference/kube-ovn-api/#vpcnattoleration","title":"VpcNatToleration","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 key String \u5bb9\u5fcd\u6c61\u70b9\u7684 key \u4fe1\u606f operator String \u53d6\u503c\u4e3a <code>Exists</code> \u6216\u8005 <code>Equal</code> value String \u5bb9\u5fcd\u6c61\u70b9\u7684 value \u4fe1\u606f effect String \u5bb9\u5fcd\u6c61\u70b9\u7684\u4f5c\u7528\u6548\u679c\uff0c\u53d6\u503c\u4e3a <code>NoExecute</code> \u3001<code>NoSchedule</code> \u6216\u8005 <code>PreferNoSchedule</code> tolerationSeconds Int64 \u6dfb\u52a0\u6c61\u70b9\u540e\uff0cPod \u8fd8\u80fd\u7ee7\u7eed\u5728\u8282\u70b9\u4e0a\u8fd0\u884c\u7684\u65f6\u95f4 <p>\u4ee5\u4e0a\u5bb9\u5fcd\u5b57\u6bb5\u7684\u542b\u4e49\uff0c\u53ef\u4ee5\u53c2\u8003 Kubernetes \u5b98\u65b9\u6587\u6863 \u6c61\u70b9\u548c\u5bb9\u5fcd\u5ea6\u3002</p>"},{"location":"reference/kube-ovn-api/#iptableseip","title":"IptablesEIP","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>IptablesEIP</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec IptablesEipSpec Vpc \u7f51\u5173\u4f7f\u7528\u7684 IptablesEIP \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status IptablesEipStatus Vpc \u7f51\u5173\u4f7f\u7528\u7684 IptablesEIP \u72b6\u6001\u4fe1\u606f"},{"location":"reference/kube-ovn-api/#iptableseipspec","title":"IptablesEipSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 v4ip String IptablesEIP v4 \u5730\u5740 v6ip String IptablesEIP v6 \u5730\u5740 macAddress String IptablesEIP crd \u8bb0\u5f55\u5206\u914d\u7684 mac \u5730\u5740\uff0c\u6ca1\u6709\u5b9e\u9645\u4f7f\u7528 natGwDp String Vpc \u7f51\u5173\u540d\u79f0"},{"location":"reference/kube-ovn-api/#iptableseipstatus","title":"IptablesEipStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 ready Bool IptablesEIP \u662f\u5426\u914d\u7f6e\u5b8c\u6210 ip String IptablesEIP \u4f7f\u7528\u7684 IP \u5730\u5740\uff0c\u76ee\u524d\u53ea\u652f\u6301\u4e86 IPv4 \u5730\u5740 redo String IptablesEIP crd \u521b\u5efa\u6216\u8005\u66f4\u65b0\u65f6\u95f4 nat String IptablesEIP \u7684\u4f7f\u7528\u7c7b\u578b\uff0c\u53d6\u503c\u4e3a <code>fip</code>\u3001<code>snat</code> \u6216\u8005 <code>dnat</code> conditions []IptablesEIPCondition IptablesEIP \u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49"},{"location":"reference/kube-ovn-api/#iptablesfiprule","title":"IptablesFIPRule","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>IptablesFIPRule</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec IptablesFIPRuleSpec Vpc \u7f51\u5173\u4f7f\u7528\u7684 IptablesFIPRule \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status IptablesFIPRuleStatus Vpc \u7f51\u5173\u4f7f\u7528\u7684 IptablesFIPRule \u72b6\u6001\u4fe1\u606f"},{"location":"reference/kube-ovn-api/#iptablesfiprulespec","title":"IptablesFIPRuleSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 eip String IptablesFIPRule \u4f7f\u7528\u7684 IptablesEIP \u540d\u79f0 internalIp String IptablesFIPRule \u5bf9\u5e94\u7684\u5185\u90e8\u7684 IP \u5730\u5740"},{"location":"reference/kube-ovn-api/#iptablesfiprulestatus","title":"IptablesFIPRuleStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 ready Bool IptablesFIPRule \u662f\u5426\u914d\u7f6e\u5b8c\u6210 v4ip String IptablesEIP \u4f7f\u7528\u7684 v4 IP \u5730\u5740 v6ip String IptablesEIP \u4f7f\u7528\u7684 v6 IP \u5730\u5740 natGwDp String Vpc \u7f51\u5173\u540d\u79f0 redo String IptablesFIPRule crd \u521b\u5efa\u6216\u8005\u66f4\u65b0\u65f6\u95f4 conditions []IptablesFIPRuleCondition IptablesFIPRule \u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49"},{"location":"reference/kube-ovn-api/#iptablessnatrule","title":"IptablesSnatRule","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>IptablesSnatRule</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec IptablesSnatRuleSpec Vpc \u7f51\u5173\u4f7f\u7528\u7684 IptablesSnatRule \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status IptablesSnatRuleStatus Vpc \u7f51\u5173\u4f7f\u7528\u7684 IptablesSnatRule \u72b6\u6001\u4fe1\u606f"},{"location":"reference/kube-ovn-api/#iptablessnatrulespec","title":"IptablesSnatRuleSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 eip String IptablesSnatRule \u4f7f\u7528\u7684 IptablesEIP \u540d\u79f0 internalIp String IptablesSnatRule \u5bf9\u5e94\u7684\u5185\u90e8\u7684 IP \u5730\u5740"},{"location":"reference/kube-ovn-api/#iptablessnatrulestatus","title":"IptablesSnatRuleStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 ready Bool IptablesSnatRule \u662f\u5426\u914d\u7f6e\u5b8c\u6210 v4ip String IptablesSnatRule \u4f7f\u7528\u7684 v4 IP \u5730\u5740 v6ip String IptablesSnatRule \u4f7f\u7528\u7684 v6 IP \u5730\u5740 natGwDp String Vpc \u7f51\u5173\u540d\u79f0 redo String IptablesSnatRule crd \u521b\u5efa\u6216\u8005\u66f4\u65b0\u65f6\u95f4 conditions []IptablesSnatRuleCondition IptablesSnatRule \u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49"},{"location":"reference/kube-ovn-api/#iptablesdnatrule","title":"IptablesDnatRule","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>IptablesDnatRule</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec IptablesDnatRuleSpec Vpc \u7f51\u5173\u4f7f\u7528\u7684 IptablesDnatRule \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status IptablesDnatRuleStatus Vpc \u7f51\u5173\u4f7f\u7528\u7684 IptablesDnatRule \u72b6\u6001\u4fe1\u606f"},{"location":"reference/kube-ovn-api/#iptablesdnatrulespec","title":"IptablesDnatRuleSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 eip Sting Vpc \u7f51\u5173\u914d\u7f6e IptablesDnatRule \u4f7f\u7528\u7684 IptablesEIP \u540d\u79f0 externalPort Sting Vpc \u7f51\u5173\u914d\u7f6e IptablesDnatRule \u4f7f\u7528\u7684\u5916\u90e8\u7aef\u53e3 protocol Sting Vpc \u7f51\u5173\u914d\u7f6e IptablesDnatRule \u7684\u534f\u8bae\u7c7b\u578b internalIp Sting Vpc \u7f51\u5173\u914d\u7f6e IptablesDnatRule \u4f7f\u7528\u7684\u5185\u90e8 IP \u5730\u5740 internalPort Sting Vpc \u7f51\u5173\u914d\u7f6e IptablesDnatRule \u4f7f\u7528\u7684\u5185\u90e8\u7aef\u53e3"},{"location":"reference/kube-ovn-api/#iptablesdnatrulestatus","title":"IptablesDnatRuleStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 ready Bool IptablesDnatRule \u662f\u5426\u914d\u7f6e\u5b8c\u6210 v4ip String IptablesDnatRule \u4f7f\u7528\u7684 v4 IP \u5730\u5740 v6ip String IptablesDnatRule \u4f7f\u7528\u7684 v6 IP \u5730\u5740 natGwDp String Vpc \u7f51\u5173\u540d\u79f0 redo String IptablesDnatRule crd \u521b\u5efa\u6216\u8005\u66f4\u65b0\u65f6\u95f4 conditions []IptablesDnatRuleCondition IptablesDnatRule \u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49"},{"location":"reference/kube-ovn-api/#vpcdns","title":"VpcDns","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>VpcDns</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec VpcDnsSpec VpcDns \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status VpcDnsStatus VpcDns \u72b6\u6001\u4fe1\u606f"},{"location":"reference/kube-ovn-api/#vpcdnsspec","title":"VpcDnsSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 vpc String VpcDns \u6240\u5728\u7684 Vpc \u540d\u79f0 subnet String VpcDns Pod \u5206\u914d\u5730\u5740\u7684 Subnet \u540d\u79f0"},{"location":"reference/kube-ovn-api/#vpcdnsstatus","title":"VpcDnsStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 conditions []VpcDnsCondition VpcDns \u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49 active Bool VpcDns \u662f\u5426\u6b63\u5728\u4f7f\u7528 <p>VpcDns \u7684\u8be6\u7ec6\u4f7f\u7528\u6587\u6863\uff0c\u53ef\u4ee5\u53c2\u8003 \u81ea\u5b9a\u4e49 VPC DNS\u3002</p>"},{"location":"reference/kube-ovn-api/#switchlbrule","title":"SwitchLBRule","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>SwitchLBRule</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec SwitchLBRuleSpec SwitchLBRule \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status SwitchLBRuleStatus SwitchLBRule \u72b6\u6001\u4fe1\u606f"},{"location":"reference/kube-ovn-api/#switchlbrulespec","title":"SwitchLBRuleSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 vip String SwitchLBRule \u914d\u7f6e\u7684 vip \u5730\u5740 namespace String SwitchLBRule \u7684\u547d\u540d\u7a7a\u95f4 selector []String \u6807\u51c6 Kubernetes Selector \u5339\u914d\u4fe1\u606f sessionAffinity String \u6807\u51c6 Kubernetes Service \u4e2d sessionAffinity \u53d6\u503c ports []SlrPort SwitchLBRule \u7aef\u53e3\u5217\u8868 <p>SwitchLBRule \u7684\u8be6\u7ec6\u914d\u7f6e\u4fe1\u606f\uff0c\u53ef\u4ee5\u53c2\u8003 \u81ea\u5b9a\u4e49 VPC \u5185\u90e8\u8d1f\u8f7d\u5747\u8861\u3002</p>"},{"location":"reference/kube-ovn-api/#slrport","title":"SlrPort","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 name String \u7aef\u53e3\u540d\u79f0 port Int32 \u7aef\u53e3\u53f7 targetPort Int32 \u76ee\u6807\u7aef\u53e3\u53f7 protocol String \u534f\u8bae\u7c7b\u578b"},{"location":"reference/kube-ovn-api/#switchlbrulestatus","title":"SwitchLBRuleStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 conditions []SwitchLBRuleCondition SwitchLBRule \u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49 ports String SwitchLBRule \u7aef\u53e3\u4fe1\u606f service String SwitchLBRule \u63d0\u4f9b\u670d\u52a1\u7684 service \u540d\u79f0"},{"location":"reference/kube-ovn-api/#vip","title":"\u5b89\u5168\u7ec4\u4e0e Vip","text":""},{"location":"reference/kube-ovn-api/#securitygroup","title":"SecurityGroup","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>SecurityGroup</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec SecurityGroupSpec \u5b89\u5168\u7ec4\u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status SecurityGroupStatus \u5b89\u5168\u7ec4\u72b6\u6001\u4fe1\u606f"},{"location":"reference/kube-ovn-api/#securitygroupspec","title":"SecurityGroupSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 ingressRules []*SgRule \u5165\u65b9\u5411\u5b89\u5168\u7ec4\u89c4\u5219 egressRules []*SgRule \u51fa\u65b9\u5411\u5b89\u5168\u7ec4\u89c4\u5219 allowSameGroupTraffic Bool \u540c\u4e00\u5b89\u5168\u7ec4\u5185\u7684 lsp \u662f\u5426\u53ef\u4ee5\u4e92\u901a\uff0c\u4ee5\u53ca\u6d41\u91cf\u89c4\u5219\u662f\u5426\u9700\u8981\u66f4\u65b0"},{"location":"reference/kube-ovn-api/#sgrule","title":"SgRule","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 ipVersion String IP \u7248\u672c\u53f7\uff0c\u53d6\u503c\u4e3a <code>ipv4</code> \u6216\u8005 <code>ipv6</code> protocol String \u53d6\u503c\u4e3a <code>all</code>\u3001<code>icmp</code>\u3001<code>tcp</code> \u6216\u8005 <code>udp</code> priority Int Acl \u4f18\u5148\u7ea7\uff0c\u53d6\u503c\u8303\u56f4\u4e3a 1-200\uff0c\u6570\u503c\u8d8a\u5c0f\uff0c\u4f18\u5148\u7ea7\u8d8a\u9ad8 remoteType String \u53d6\u503c\u4e3a <code>address</code> \u6216\u8005 <code>securityGroup</code> remoteAddress String \u5bf9\u7aef\u5730\u5740 remoteSecurityGroup String \u5bf9\u7aef\u5b89\u5168\u7ec4 portRangeMin Int \u7aef\u53e3\u8303\u56f4\u8d77\u59cb\u503c\uff0c\u6700\u5c0f\u53d6\u503c\u4e3a 1 portRangeMax Int \u7aef\u53e3\u8303\u56f4\u6700\u5927\u503c\uff0c\u6700\u5927\u53d6\u503c\u4e3a 65535 policy String \u53d6\u503c\u4e3a <code>allow</code> \u6216\u8005 <code>drop</code>"},{"location":"reference/kube-ovn-api/#securitygroupstatus","title":"SecurityGroupStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 portGroup String \u5b89\u5168\u7ec4\u5bf9\u5e94\u7684 port-group \u540d\u79f0 allowSameGroupTraffic Bool \u540c\u4e00\u5b89\u5168\u7ec4\u5185\u7684 lsp \u662f\u5426\u53ef\u4ee5\u4e92\u901a\uff0c\u4ee5\u53ca\u5b89\u5168\u7ec4\u7684\u6d41\u91cf\u89c4\u5219\u662f\u5426\u9700\u8981\u66f4\u65b0 ingressMd5 String \u5165\u65b9\u5411\u5b89\u5168\u7ec4\u89c4\u5219 MD5 \u53d6\u503c egressMd5 String \u51fa\u65b9\u5411\u5b89\u5168\u7ec4\u89c4\u5219 MD5 \u53d6\u503c ingressLastSyncSuccess Bool \u5165\u65b9\u5411\u89c4\u5219\u4e0a\u4e00\u6b21\u540c\u6b65\u662f\u5426\u6210\u529f egressLastSyncSuccess Bool \u51fa\u65b9\u5411\u89c4\u5219\u4e0a\u4e00\u6b21\u540c\u6b65\u662f\u5426\u6210\u529f"},{"location":"reference/kube-ovn-api/#vip_1","title":"Vip","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>Vip</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec VipSpec Vip \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status VipStatus Vip \u72b6\u6001\u4fe1\u606f"},{"location":"reference/kube-ovn-api/#vipspec","title":"VipSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 namespace String Vip \u6240\u5728\u547d\u540d\u7a7a\u95f4 subnet String Vip \u6240\u5c5e\u5b50\u7f51 v4ip String Vip v4 IP \u5730\u5740 v6ip String Vip v6 IP \u5730\u5740 macAddress String Vip mac \u5730\u5740 parentV4ip String \u76ee\u524d\u6ca1\u6709\u4f7f\u7528 parentV6ip String \u76ee\u524d\u6ca1\u6709\u4f7f\u7528 parentMac String \u76ee\u524d\u6ca1\u6709\u4f7f\u7528 attachSubnets []String \u8be5\u5b57\u6bb5\u5e9f\u5f03\uff0c\u4e0d\u518d\u4f7f\u7528"},{"location":"reference/kube-ovn-api/#vipstatus","title":"VipStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 conditions []VipCondition Vip \u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49 ready Bool Vip \u662f\u5426\u51c6\u5907\u597d v4ip String Vip v4 IP \u5730\u5740\uff0c\u5e94\u8be5\u548c spec \u5b57\u6bb5\u53d6\u503c\u4e00\u81f4 v6ip String Vip v6 IP \u5730\u5740\uff0c\u5e94\u8be5\u548c spec \u5b57\u6bb5\u53d6\u503c\u4e00\u81f4 mac String Vip mac \u5730\u5740\uff0c\u5e94\u8be5\u548c spec \u5b57\u6bb5\u53d6\u503c\u4e00\u81f4 pv4ip String \u76ee\u524d\u6ca1\u6709\u4f7f\u7528 pv6ip String \u76ee\u524d\u6ca1\u6709\u4f7f\u7528 pmac String \u76ee\u524d\u6ca1\u6709\u4f7f\u7528"},{"location":"reference/kube-ovn-api/#ovneip","title":"OvnEip","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>OvnEip</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec OvnEipSpec \u9ed8\u8ba4 Vpc \u4f7f\u7528 OvnEip \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status OvnEipStatus \u9ed8\u8ba4 Vpc \u4f7f\u7528 OvnEip \u72b6\u6001\u4fe1\u606f"},{"location":"reference/kube-ovn-api/#ovneipspec","title":"OvnEipSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 externalSubnet String OvnEip \u6240\u5728\u7684\u5b50\u7f51\u540d\u79f0 v4ip String OvnEip IP \u5730\u5740 macAddress String OvnEip Mac \u5730\u5740 type String OvnEip \u4f7f\u7528\u7c7b\u578b\uff0c\u53d6\u503c\u6709 <code>fip</code>\u3001<code>snat</code> \u6216\u8005 <code>lrp</code>"},{"location":"reference/kube-ovn-api/#ovneipstatus","title":"OvnEipStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 conditions []OvnEipCondition \u9ed8\u8ba4 Vpc OvnEip \u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49 v4ip String OvnEip \u4f7f\u7528\u7684 v4 IP \u5730\u5740 macAddress String OvnEip \u4f7f\u7528\u7684 Mac \u5730\u5740"},{"location":"reference/kube-ovn-api/#ovnfip","title":"OvnFip","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>OvnFip</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec OvnFipSpec \u9ed8\u8ba4 Vpc \u4f7f\u7528 OvnFip \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status OvnFipStatus \u9ed8\u8ba4 Vpc \u4f7f\u7528 OvnFip \u72b6\u6001\u4fe1\u606f"},{"location":"reference/kube-ovn-api/#ovnfipspec","title":"OvnFipSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 ovnEip String OvnFip \u7ed1\u5b9a\u7684 OvnEip \u540d\u79f0 ipName String OvnFip \u7ed1\u5b9a Pod \u5bf9\u5e94\u7684 IP crd \u540d\u79f0"},{"location":"reference/kube-ovn-api/#ovnfipstatus","title":"OvnFipStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 ready Bool OvnFip \u662f\u5426\u914d\u7f6e\u5b8c\u6210 v4Eip String OvnFip \u7ed1\u5b9a\u7684 OvnEip \u540d\u79f0 v4Ip String OvnFip \u5f53\u524d\u4f7f\u7528\u7684 OvnEip \u5730\u5740 macAddress String OvnFip \u914d\u7f6e\u7684 Mac \u5730\u5740 vpc String OvnFip \u6240\u5728\u7684 Vpc \u540d\u79f0 conditions []OvnFipCondition OvnFip \u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49"},{"location":"reference/kube-ovn-api/#ovnsnatrule","title":"OvnSnatRule","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>OvnSnatRule</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec OvnSnatRuleSpec \u9ed8\u8ba4 Vpc OvnSnatRule \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status OvnSnatRuleStatus \u9ed8\u8ba4 Vpc OvnSnatRule \u72b6\u6001\u4fe1\u606f"},{"location":"reference/kube-ovn-api/#ovnsnatrulespec","title":"OvnSnatRuleSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 ovnEip String OvnSnatRule \u7ed1\u5b9a\u7684 OvnEip \u540d\u79f0 vpcSubnet String OvnSnatRule \u914d\u7f6e\u7684\u5b50\u7f51\u540d\u79f0 ipName String OvnSnatRule \u7ed1\u5b9a Pod \u5bf9\u5e94\u7684 IP crd \u540d\u79f0"},{"location":"reference/kube-ovn-api/#ovnsnatrulestatus","title":"OvnSnatRuleStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 ready Bool OvnSnatRule \u662f\u5426\u914d\u7f6e\u5b8c\u6210 v4Eip String OvnSnatRule \u7ed1\u5b9a\u7684 OvnEip \u5730\u5740 v4IpCidr String \u5728 logical-router \u4e2d\u914d\u7f6e snat \u8f6c\u6362\u4f7f\u7528\u7684 cidr \u5730\u5740 vpc String OvnSnatRule \u6240\u5728\u7684 Vpc \u540d\u79f0 conditions []OvnSnatRuleCondition OvnSnatRule \u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49 <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"reference/kube-ovn-pinger-args/","title":"Kube-OVN-Pinger \u53c2\u6570\u53c2\u8003","text":"<p>\u57fa\u4e8e Kube-OVN v1.12.0 \u7248\u672c\uff0c\u6574\u7406\u4e86 Kube-ovn-pinger \u652f\u6301\u7684\u53c2\u6570\uff0c\u5217\u51fa\u53c2\u6570\u5b9a\u4e49\u5404\u5b57\u6bb5\u7684\u53d6\u503c\u7c7b\u578b\uff0c\u542b\u4e49\u548c\u9ed8\u8ba4\u503c\uff0c\u4ee5\u4f9b\u53c2\u8003</p>"},{"location":"reference/kube-ovn-pinger-args/#_1","title":"\u53c2\u6570\u63cf\u8ff0","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 \u9ed8\u8ba4\u503c port Int metrics \u7aef\u53e3 8080 kubeconfig String \u5177\u6709\u8ba4\u8bc1\u4fe1\u606f\u7684 kubeconfig \u6587\u4ef6\u8def\u5f84\uff0c \u5982\u679c\u672a\u8bbe\u7f6e\uff0c\u4f7f\u7528 inCluster \u4ee4\u724c\u3002 \"\" ds-namespace String kube-ovn-pinger \u5b88\u62a4\u8fdb\u7a0b\u547d\u540d\u7a7a\u95f4 \"kube-system\" ds-name String kube-ovn-pinger \u5b88\u62a4\u8fdb\u7a0b\u540d\u5b57 \"kube-ovn-pinger\" interval Int \u8fde\u7eed ping \u4e4b\u95f4\u7684\u95f4\u9694\u79d2\u6570 5 mode String \u670d\u52a1\u5668\u6216\u5de5\u4f5c\u6a21\u5f0f \"server\" exit-code Int \u5931\u8d25\u65f6\u9000\u51fa\u4ee3\u7801 0 internal-dns String \u4ece pod \u5185\u89e3\u6790\u5185\u90e8 dns \"kubernetes.default\" external-dns String \u4ece pod \u5185\u89e3\u6790\u5916\u90e8 dns \"\" external-address String \u68c0\u67e5\u4e0e\u5916\u90e8\u5730\u5740\u7684 ping \u8fde\u901a \"114.114.114.114\" network-mode String \u5f53\u524d\u96c6\u7fa4\u4f7f\u7528\u7684 cni \u63d2\u4ef6 \"kube-ovn\" enable-metrics Bool \u662f\u5426\u652f\u6301 metrics \u67e5\u8be2 true ovs.timeout Int \u5bf9 OVS \u7684 JSON-RPC \u8bf7\u6c42\u8d85\u65f6\u3002 2 system.run.dir String OVS \u9ed8\u8ba4\u8fd0\u884c\u76ee\u5f55\u3002 \"/var/run/openvswitch\" database.vswitch.name String OVS \u6570\u636e\u5e93\u7684\u540d\u79f0\u3002 \"Open_vSwitch\" database.vswitch.socket.remote String JSON-RPC unix \u5957\u63a5\u5b57\u5230 OVS \u6570\u636e\u5e93\u3002 \"unix:/var/run/openvswitch/db.sock\" database.vswitch.file.data.path String OVS \u6570\u636e\u5e93\u6587\u4ef6\u3002 \"/etc/openvswitch/conf.db\" database.vswitch.file.log.path String OVS \u6570\u636e\u5e93\u65e5\u5fd7\u6587\u4ef6\u3002 \"/var/log/openvswitch/ovsdb-server.log\" database.vswitch.file.pid.path String OVS \u6570\u636e\u5e93\u8fdb\u7a0b ID \u6587\u4ef6\u3002 \"/var/run/openvswitch/ovsdb-server.pid\" database.vswitch.file.system.id.path String OVS \u7cfb\u7edf\u6807\u8bc6\u6587\u4ef6\u3002 \"/etc/openvswitch/system-id.conf\" service.vswitchd.file.log.path String OVS vswitchd \u5b88\u62a4\u8fdb\u7a0b\u65e5\u5fd7\u6587\u4ef6\u3002 \"/var/log/openvswitch/ovs-vswitchd.log\" service.vswitchd.file.pid.path String OVS vswitchd \u5b88\u62a4\u8fdb\u7a0b\u8fdb\u7a0b ID \u6587\u4ef6\u3002 \"/var/run/openvswitch/ovs-vswitchd.pid\" service.ovncontroller.file.log.path String OVN \u63a7\u5236\u5668\u5b88\u62a4\u8fdb\u7a0b\u65e5\u5fd7\u6587\u4ef6\u3002 \"/var/log/ovn/ovn-controller.log\" service.ovncontroller.file.pid.path String OVN \u63a7\u5236\u5668\u5b88\u62a4\u8fdb\u7a0b\u8fdb\u7a0b ID \u6587\u4ef6\u3002 \"/var/run/ovn/ovn-controller.pid\" <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"reference/metrics/","title":"Kube-OVN \u76d1\u63a7\u6307\u6807","text":"<p>\u672c\u6587\u6863\u5217\u4e3e Kube-OVN \u6240\u63d0\u4f9b\u7684\u76d1\u63a7\u6307\u6807\u3002</p>"},{"location":"reference/metrics/#ovn-monitor","title":"ovn-monitor","text":"<p>OVN \u81ea\u8eab\u72b6\u6001\u76d1\u63a7\u6307\u6807\uff1a</p> \u7c7b\u578b \u6307\u6807\u9879 \u63cf\u8ff0 Gauge kube_ovn_ovn_status OVN \u89d2\u8272\u72b6\u6001\uff0c (2) \u4e3a follower\uff1b (1) \u4e3a leader, (0) \u4e3a\u5f02\u5e38\u72b6\u6001\u3002 Gauge kube_ovn_failed_req_count OVN \u5931\u8d25\u8bf7\u6c42\u6570\u91cf\u3002 Gauge kube_ovn_log_file_size_bytes OVN \u7ec4\u4ef6\u65e5\u5fd7\u6587\u4ef6\u5927\u5c0f\u3002 Gauge kube_ovn_db_file_size_bytes OVN \u7ec4\u4ef6\u6570\u636e\u5e93\u6587\u4ef6\u5927\u5c0f\u3002 Gauge kube_ovn_chassis_info OVN chassis \u72b6\u6001 (1) \u8fd0\u884c\u4e2d\uff0c(0) \u505c\u6b62\u3002 Gauge kube_ovn_db_status OVN \u6570\u636e\u5e93\u72b6\u6001, (1) \u4e3a\u6b63\u5e38\uff1b (0) \u4e3a\u5f02\u5e38\u3002 Gauge kube_ovn_logical_switch_info OVN logical switch \u4fe1\u606f\uff0c\u503c\u4e3a (1)\uff0c\u6807\u7b7e\u4e2d\u5305\u542b logical switch \u540d\u5b57\u3002 Gauge kube_ovn_logical_switch_external_id OVN logical switch external_id \u4fe1\u606f\uff0c\u503c\u4e3a (1)\uff0c\u6807\u7b7e\u4e2d\u5305\u542b external-id \u5185\u5bb9\u3002 Gauge kube_ovn_logical_switch_port_binding OVN logical switch \u548c logical switch port \u5173\u8054\u4fe1\u606f\uff0c\u503c\u4e3a (1)\uff0c\u901a\u8fc7\u6807\u7b7e\u8fdb\u884c\u5173\u8054\u3002 Gauge kube_ovn_logical_switch_tunnel_key \u548c OVN logical switch \u5173\u8054\u7684 tunnel key \u4fe1\u606f\u3002 Gauge kube_ovn_logical_switch_ports_num OVN logical switch \u4e0a logical port \u7684\u6570\u91cf\u3002 Gauge kube_ovn_logical_switch_port_info OVN logical switch port \u4fe1\u606f\uff0c\u503c\u4e3a (1)\uff0c\u6807\u7b7e\u4e2d\u5305\u542b\u5177\u4f53\u4fe1\u606f\u3002 Gauge kube_ovn_logical_switch_port_tunnel_key \u548c OVN logical switch port \u5173\u8054\u7684 tunnel key \u4fe1\u606f\u3002 Gauge kube_ovn_cluster_enabled (1) OVN \u6570\u636e\u5e93\u4e3a\u96c6\u7fa4\u6a21\u5f0f\uff1b (0) OVN \u6570\u636e\u5e93\u4e3a\u975e\u96c6\u7fa4\u6a21\u5f0f\u3002 Gauge kube_ovn_cluster_role \u6bcf\u4e2a\u6570\u636e\u5e93\u5b9e\u4f8b\u7684\u89d2\u8272\uff0c\u503c\u4e3a (1)\uff0c\u6807\u7b7e\u4e2d\u5305\u542b\u5bf9\u5e94\u89d2\u8272\u4fe1\u606f\u3002 Gauge kube_ovn_cluster_status \u6bcf\u4e2a\u6570\u636e\u5e93\u5b9e\u4f8b\u7684\u72b6\u6001\uff0c\u503c\u4e3a (1)\uff0c\u6807\u7b7e\u4e2d\u5305\u542b\u5bf9\u5e94\u72b6\u6001\u4fe1\u606f\u3002 Gauge kube_ovn_cluster_term RAFT term \u4fe1\u606f\u3002 Gauge kube_ovn_cluster_leader_self \u5f53\u524d\u6570\u636e\u5e93\u5b9e\u4f8b\u662f\u5426\u4e3a leader (1) \u662f\uff0c (0) \u4e0d\u662f\u3002 Gauge kube_ovn_cluster_vote_self \u5f53\u524d\u6570\u636e\u5e93\u5b9e\u4f8b\u662f\u5426\u9009\u4e3e\u81ea\u5df1\u4e3a leader (1) \u662f\uff0c (0) \u4e0d\u662f\u3002 Gauge kube_ovn_cluster_election_timer \u5f53\u524d election timer \u503c\u3002 Gauge kube_ovn_cluster_log_not_committed \u672a commit \u7684 RAFT \u65e5\u5fd7\u6570\u91cf\u3002 Gauge kube_ovn_cluster_log_not_applied \u672a apply \u7684 RAFT \u65e5\u5fd7\u6570\u91cf\u3002 Gauge kube_ovn_cluster_log_index_start \u5f53\u524d RAFT \u65e5\u5fd7\u6761\u76ee\u7684\u8d77\u59cb\u503c\u3002 Gauge kube_ovn_cluster_log_index_next RAFT \u65e5\u5fd7\u6761\u76ee\u7684\u4e0b\u4e00\u4e2a\u503c\u3002 Gauge kube_ovn_cluster_inbound_connections_total \u5f53\u524d\u5b9e\u4f8b\u7684\u5165\u5411\u8fde\u63a5\u6570\u91cf\u3002 Gauge kube_ovn_cluster_outbound_connections_total \u5f53\u524d\u5b9e\u4f8b\u7684\u51fa\u5411\u8fde\u63a5\u6570\u91cf\u3002 Gauge kube_ovn_cluster_inbound_connections_error_total \u5f53\u524d\u5b9e\u4f8b\u7684\u5165\u5411\u9519\u8bef\u8fde\u63a5\u6570\u91cf\u3002 Gauge kube_ovn_cluster_outbound_connections_error_total \u5f53\u524d\u5b9e\u4f8b\u7684\u51fa\u5411\u9519\u8bef\u8fde\u63a5\u6570\u91cf\u3002"},{"location":"reference/metrics/#ovs-monitor","title":"ovs-monitor","text":"<p><code>ovsdb</code> \u548c <code>vswitchd</code> \u81ea\u8eab\u72b6\u6001\u76d1\u63a7\u6307\u6807\uff1a</p> \u7c7b\u578b \u6307\u6807\u9879 \u63cf\u8ff0 Gauge ovs_status OVS \u5065\u5eb7\u72b6\u6001\uff0c (1) \u4e3a\u6b63\u5e38\uff0c(0) \u4e3a\u5f02\u5e38\u3002 Gauge ovs_info OVS \u57fa\u7840\u4fe1\u606f\uff0c\u503c\u4e3a (1)\uff0c\u6807\u7b7e\u4e2d\u5305\u542b\u5bf9\u5e94\u4fe1\u606f\u3002 Gauge failed_req_count OVS \u5931\u8d25\u8bf7\u6c42\u6570\u91cf\u3002 Gauge log_file_size OVS \u7ec4\u4ef6\u65e5\u5fd7\u6587\u4ef6\u5927\u5c0f\u3002 Gauge db_file_size OVS \u7ec4\u4ef6\u6570\u636e\u5e93\u6587\u4ef6\u5927\u5c0f\u3002 Gauge datapath Datapath \u57fa\u7840\u4fe1\u606f\uff0c\u503c\u4e3a (1)\uff0c\u6807\u7b7e\u4e2d\u5305\u542b\u5bf9\u5e94\u4fe1\u606f\u3002 Gauge dp_total \u5f53\u524d OVS \u4e2d datapath \u6570\u91cf\u3002 Gauge dp_if Datapath \u63a5\u53e3\u57fa\u7840\u4fe1\u606f\uff0c\u503c\u4e3a (1)\uff0c\u6807\u7b7e\u4e2d\u5305\u542b\u5bf9\u5e94\u4fe1\u606f\u3002 Gauge dp_if_total \u5f53\u524d datapath \u4e2d port \u6570\u91cf\u3002 Gauge dp_flows_total Datapath \u4e2d flow \u6570\u91cf\u3002 Gauge dp_flows_lookup_hit Datapath \u4e2d\u547d\u4e2d\u5f53\u524d flow \u6570\u636e\u5305\u6570\u91cf\u3002 Gauge dp_flows_lookup_missed Datapath \u4e2d\u672a\u547d\u4e2d\u5f53\u524d flow \u6570\u636e\u5305\u6570\u91cf\u3002 Gauge dp_flows_lookup_lost Datapath \u4e2d\u9700\u8981\u53d1\u9001\u7ed9 userspace \u5904\u7406\u7684\u6570\u636e\u5305\u6570\u91cf\u3002 Gauge dp_masks_hit Datapath \u4e2d\u547d\u4e2d\u5f53\u524d mask \u6570\u636e\u5305\u6570\u91cf\u3002 Gauge dp_masks_total Datapath \u4e2d mask \u7684\u6570\u91cf\u3002 Gauge dp_masks_hit_ratio Datapath \u4e2d \u6570\u636e\u5305\u547d\u4e2d mask \u7684\u6bd4\u7387\u3002 Gauge interface OVS \u63a5\u53e3\u57fa\u7840\u4fe1\u606f\uff0c\u503c\u4e3a (1)\uff0c\u6807\u7b7e\u4e2d\u5305\u542b\u5bf9\u5e94\u4fe1\u606f\u3002 Gauge interface_admin_state \u63a5\u53e3\u7ba1\u7406\u72b6\u6001\u4fe1\u606f (0) \u4e3a down, (1) \u4e3a up, (2) \u4e3a\u5176\u4ed6\u72b6\u6001\u3002 Gauge interface_link_state \u63a5\u53e3\u94fe\u8def\u72b6\u6001\u4fe1\u606f (0) \u4e3a down, (1) \u4e3a up, (2) \u4e3a\u5176\u4ed6\u72b6\u6001\u3002 Gauge interface_mac_in_use OVS Interface \u4f7f\u7528\u7684 MAC \u5730\u5740 Gauge interface_mtu OVS Interface \u4f7f\u7528\u7684 MTU\u3002 Gauge interface_of_port OVS Interface \u5173\u8054\u7684 OpenFlow Port ID\u3002 Gauge interface_if_index OVS Interface \u5173\u8054\u7684 Index\u3002 Gauge interface_tx_packets OVS Interface \u53d1\u9001\u5305\u6570\u91cf\u3002 Gauge interface_tx_bytes OVS Interface \u53d1\u9001\u5305\u5927\u5c0f\u3002 Gauge interface_rx_packets OVS Interface \u63a5\u6536\u5305\u6570\u91cf\u3002 Gauge interface_rx_bytes OVS Interface \u63a5\u6536\u5305\u5927\u5c0f\u3002 Gauge interface_rx_crc_err OVS Interface \u63a5\u6536\u5305\u6821\u9a8c\u548c\u9519\u8bef\u6570\u91cf\u3002 Gauge interface_rx_dropped OVS Interface \u63a5\u6536\u5305\u4e22\u5f03\u6570\u91cf\u3002 Gauge interface_rx_errors OVS Interface \u63a5\u6536\u5305\u9519\u8bef\u6570\u91cf\u3002 Gauge interface_rx_frame_err OVS Interface \u63a5\u6536\u5e27\u9519\u8bef\u6570\u91cf\u3002 Gauge interface_rx_missed_err OVS Interface \u63a5\u6536\u5305 miss \u6570\u91cf\u3002 Gauge interface_rx_over_err OVS Interface \u63a5\u6536\u5305 overrun \u6570\u91cf\u3002 Gauge interface_tx_dropped OVS Interface \u53d1\u9001\u5305\u4e22\u5f03\u6570\u91cf\u3002 Gauge interface_tx_errors OVS Interface \u53d1\u9001\u5305\u9519\u8bef\u6570\u91cf\u3002 Gauge interface_collisions OVS interface \u51b2\u7a81\u6570\u91cf\u3002"},{"location":"reference/metrics/#kube-ovn-pinger","title":"kube-ovn-pinger","text":"<p>\u7f51\u7edc\u8d28\u91cf\u76f8\u5173\u76d1\u63a7\u6307\u6807\uff1a</p> \u7c7b\u578b \u6307\u6807\u9879 \u63cf\u8ff0 Gauge pinger_ovs_up \u8282\u70b9 OVS \u8fd0\u884c\u3002 Gauge pinger_ovs_down \u8282\u70b9 OVS \u505c\u6b62\u3002 Gauge pinger_ovn_controller_up \u8282\u70b9 ovn-controller \u8fd0\u884c\u3002 Gauge pinger_ovn_controller_down \u8282\u70b9 ovn-controller \u505c\u6b62\u3002 Gauge pinger_inconsistent_port_binding OVN-SB \u91cc portbinding \u6570\u91cf\u548c\u4e3b\u673a OVS interface \u4e0d\u4e00\u81f4\u7684\u6570\u91cf\u3002 Gauge pinger_apiserver_healthy kube-ovn-pinger \u53ef\u4ee5\u8054\u901a apiserver\u3002 Gauge pinger_apiserver_unhealthy kube-ovn-pinger \u65e0\u6cd5\u8054\u901a apiserver\u3002 Histogram pinger_apiserver_latency_ms kube-ovn-pinger \u8bbf\u95ee apiserver \u5ef6\u8fdf\u3002 Gauge pinger_internal_dns_healthy kube-ovn-pinger \u53ef\u4ee5\u89e3\u6790\u5185\u90e8\u57df\u540d\u3002 Gauge pinger_internal_dns_unhealthy kube-ovn-pinger \u65e0\u6cd5\u89e3\u6790\u5185\u90e8\u57df\u540d\u3002 Histogram pinger_internal_dns_latency_ms kube-ovn-pinger \u89e3\u6790\u5185\u90e8\u57df\u540d\u5ef6\u8fdf\u3002 Gauge pinger_external_dns_health kube-ovn-pinger \u53ef\u4ee5\u89e3\u6790\u5916\u90e8\u57df\u540d\u3002 Gauge pinger_external_dns_unhealthy kube-ovn-pinger \u65e0\u6cd5\u89e3\u6790\u5916\u90e8\u57df\u540d\u3002 Histogram pinger_external_dns_latency_ms kube-ovn-pinger \u89e3\u6790\u5916\u90e8\u57df\u540d\u5ef6\u8fdf\u3002 Histogram pinger_pod_ping_latency_ms kube-ovn-pinger ping Pod \u5ef6\u8fdf\u3002 Gauge pinger_pod_ping_lost_total kube-ovn-pinger ping Pod \u4e22\u5305\u6570\u91cf\u3002 Gauge pinger_pod_ping_count_total kube-ovn-pinger ping Pod \u6570\u91cf\u3002 Histogram pinger_node_ping_latency_ms kube-ovn-pinger ping Node \u5ef6\u8fdf\u3002 Gauge pinger_node_ping_lost_total kube-ovn-pinger ping Node \u4e22\u5305\u3002 Gauge pinger_node_ping_count_total kube-ovn-pinger ping Node \u6570\u91cf\u3002 Histogram pinger_external_ping_latency_ms kube-ovn-pinger ping \u5916\u90e8\u5730\u5740 \u5ef6\u8fdf\u3002 Gauge pinger_external_lost_total kube-ovn-pinger ping \u5916\u90e8\u4e22\u5305\u6570\u91cf\u3002"},{"location":"reference/metrics/#kube-ovn-controller","title":"kube-ovn-controller","text":"<p><code>kube-ovn-controller</code> \u76f8\u5173\u76d1\u63a7\u6307\u6807\uff1a</p> \u7c7b\u578b \u6307\u6807\u9879 \u63cf\u8ff0 Histogram rest_client_request_latency_seconds \u8bf7\u6c42 apiserver \u5ef6\u8fdf\u3002 Counter rest_client_requests_total \u8bf7\u6c42 apiserver \u6570\u91cf\u3002 Counter lists_total API list \u8bf7\u6c42\u6570\u91cf\u3002 Summary list_duration_seconds API list \u8bf7\u6c42\u5ef6\u8fdf\u3002 Summary items_per_list API list \u8fd4\u56de\u7ed3\u679c\u6570\u91cf\u3002 Counter watches_total API watch \u8bf7\u6c42\u6570\u91cf\u3002 Counter short_watches_total \u77ed\u65f6\u95f4 API watch \u8bf7\u6c42\u6570\u91cf\u3002 Summary watch_duration_seconds API watch \u6301\u7eed\u65f6\u95f4\u3002 Summary items_per_watch API watch \u8fd4\u56de\u7ed3\u679c\u6570\u91cf\u3002 Gauge last_resource_version \u6700\u65b0\u7684 resource version\u3002 Histogram ovs_client_request_latency_milliseconds \u8bf7\u6c42 OVN \u7ec4\u4ef6\u5ef6\u8fdf\u3002 Gauge subnet_available_ip_count \u5b50\u7f51\u53ef\u7528 IP \u6570\u91cf\u3002 Gauge subnet_used_ip_count \u5b50\u7f51\u5df2\u7528 IP \u6570\u91cf\u3002"},{"location":"reference/metrics/#kube-ovn-cni","title":"kube-ovn-cni","text":"<p><code>kube-ovn-cni</code> \u76f8\u5173\u76d1\u63a7\u6307\u6807\uff1a</p> \u7c7b\u578b \u6307\u6807\u9879 \u63cf\u8ff0 Histogram cni_op_latency_seconds CNI \u64cd\u4f5c\u5ef6\u8fdf\u3002 Counter cni_wait_address_seconds_total CNI \u7b49\u5f85\u5730\u5740\u5c31\u7eea\u65f6\u95f4\u3002 Counter cni_wait_connectivity_seconds_total CNI \u7b49\u5f85\u8fde\u63a5\u5c31\u7eea\u65f6\u95f4\u3002 Counter cni_wait_route_seconds_total CNI \u7b49\u5f85\u8def\u7531\u5c31\u7eea\u65f6\u95f4\u3002 Histogram rest_client_request_latency_seconds \u8bf7\u6c42 apiserver \u5ef6\u8fdf\u3002 Counter rest_client_requests_total \u8bf7\u6c42 apiserver \u6570\u91cf\u3002 Counter lists_total API list \u8bf7\u6c42\u6570\u91cf\u3002 Summary list_duration_seconds API list \u8bf7\u6c42\u5ef6\u8fdf\u3002 Summary items_per_list API list \u8fd4\u56de\u7ed3\u679c\u6570\u91cf\u3002 Counter watches_total API watch \u8bf7\u6c42\u6570\u91cf\u3002 Counter short_watches_total \u77ed\u65f6\u95f4 API watch \u8bf7\u6c42\u6570\u91cf\u3002 Summary watch_duration_seconds API watch \u6301\u7eed\u65f6\u95f4\u3002 Summary items_per_watch API watch \u8fd4\u56de\u7ed3\u679c\u6570\u91cf\u3002 Gauge last_resource_version \u6700\u65b0\u7684 resource version\u3002 Histogram ovs_client_request_latency_milliseconds \u8bf7\u6c42 OVN \u7ec4\u4ef6\u5ef6\u8fdf\u3002 <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"reference/ovs-ovn-customized/","title":"\u5bf9\u4e0a\u6e38 OVS/OVN \u4fee\u6539","text":"<p>\u4e0a\u6e38 OVN/OVS \u6700\u521d\u8bbe\u8ba1\u76ee\u6807\u4e3a\u901a\u7528 SDN \u63a7\u5236\u5668\u548c\u6570\u636e\u5e73\u9762\u3002\u7531\u4e8e Kubernetes \u7f51\u7edc\u5b58\u5728\u4e00\u4e9b\u7279\u6b8a\u7684\u7528\u6cd5\uff0c \u5e76\u4e14 Kube-OVN \u53ea\u91cd\u70b9\u4f7f\u7528\u4e86\u90e8\u5206\u529f\u80fd\uff0c\u4e3a\u4e86 \u8fbe\u5230\u66f4\u597d\u7684\u6027\u80fd\u3001\u7a33\u5b9a\u6027\u548c\u7279\u5b9a\u7684\u529f\u80fd\uff0cKube-OVN \u5bf9\u4e0a\u6e38 OVN/OVS \u505a\u4e86\u90e8\u5206\u4fee\u6539\u3002\u7528\u6237\u5982\u679c\u4f7f\u7528\u81ea\u5df1\u7684 OVN/OVS \u914d\u5408 Kube-OVN \u7684\u63a7\u5236\u5668\u8fdb\u884c\u5de5\u4f5c\u65f6\u9700\u8981\u6ce8\u610f \u4e0b\u8ff0\u7684\u6539\u52a8\u53ef\u80fd\u9020\u6210\u7684\u5f71\u54cd\u3002</p> <p>\u672a\u5408\u5165\u4e0a\u6e38\u4fee\u6539\uff1a</p> <ul> <li>38df6fa3f7 \u8c03\u6574\u9009\u4e3e timer\uff0c\u907f\u514d\u5927\u89c4\u6a21\u96c6\u7fa4\u9009\u4e3e\u6296\u52a8\u3002</li> <li>d4888c4e75 \u6dfb\u52a0 fdb \u66f4\u65b0\u65e5\u5fd7\u3002</li> <li>d4888c4e75 \u4fee\u590d hairpin \u73af\u5883\u4e0b fdb \u5b66\u4e60\u9519\u8bef\u7684\u95ee\u9898\u3002</li> <li>9a81b91368 \u4e3a ovsdb-tool \u7684 join-cluster \u5b50\u547d\u4ee4\u6dfb\u52a0 Server ID \u53c2\u6570\u3002</li> <li>62d4969877 \u4fee\u590d\u5f00\u542f SSL \u540e OVSDB \u76d1\u542c\u5730\u5740\u9519\u8bef\u7684\u95ee\u9898\u3002</li> <li>0700cb90f9 \u76ee\u7684\u5730\u5740\u975e Service \u6d41\u91cf\u7ed5\u8fc7 conntrack \u4ee5\u63d0\u9ad8\u7279\u5b9a\u6570\u636e\u94fe\u8def\u6027\u80fd\u3002</li> <li>c48049a64f ECMP \u7b97\u6cd5\u7531 dp_hash \u8c03\u6574\u4e3a hash\uff0c\u907f\u514d\u90e8\u5206\u5185\u6838\u51fa\u73b0\u7684\u54c8\u5e0c\u9519\u8bef\u95ee\u9898\u3002</li> <li>64383c14a9 \u4fee\u590d Windows \u4e0b\u5185\u6838 Crash \u95ee\u9898\u3002</li> <li>08a95db2ca \u652f\u6301 Windows \u4e0b\u7684 github action \u6784\u5efa\u3002</li> <li>680e77a190 Windows \u4e0b\u9ed8\u8ba4\u4f7f\u7528 tcp \u76d1\u542c\u3002</li> <li>05e57b3227 \u652f\u6301 Windows \u7f16\u8bd1\u3002</li> <li>b3801ecb73 \u4fee\u6539\u6e90\u8def\u7531\u7684\u4f18\u5148\u7ea7\u3002</li> <li>977e569539 \u4fee\u590d Underlay \u6a21\u5f0f\u4e0b Pod \u6570\u91cf\u8fc7\u591a\u5bfc\u81f4 OVS \u6d41\u8868 resubmit \u6b21\u6570\u8d85\u8fc7\u4e0a\u9650\u7684\u95ee\u9898\u3002</li> <li>45a4a22161 ovn-nbctl\uff1avips \u4e3a\u7a7a\u65f6\u4e0d\u5220\u9664 Load Balancer\u3002</li> <li>540592b9ff DNAT \u540e\u66ff\u6362 Mac \u5730\u5740\u4e3a\u76ee\u6807\u5730\u5740\uff0c\u51cf\u5c11\u989d\u5916\u6027\u80fd\u5f00\u9500\u3002</li> <li>10972d9632 \u4fee\u590d vswitchd ofport_usage \u5185\u5b58\u6cc4\u9732\u3002</li> </ul> <p>\u5df2\u5408\u5165\u4e0a\u6e38\u4fee\u6539\uff1a</p> <ul> <li>20626ea909 \u7ec4\u64ad\u6d41\u91cf\u7ed5\u8fc7 LB \u548c ACL \u5904\u7406\u9636\u6bb5\uff0c\u4ee5\u63d0\u9ad8\u7279\u5b9a\u6570\u636e\u94fe\u8def\u6027\u80fd\u3002</li> <li>a2d9ff3ccd Deb \u6784\u5efa\u589e\u52a0\u7f16\u8bd1\u4f18\u5316\u9009\u9879\u3002</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"reference/tunnel-protocol/","title":"\u96a7\u9053\u534f\u8bae\u8bf4\u660e","text":"<p>Kube-OVN \u4f7f\u7528 OVN/OVS \u4f5c\u4e3a\u6570\u636e\u5e73\u9762\u5b9e\u73b0\uff0c\u76ee\u524d\u652f\u6301 <code>Geneve</code>\uff0c<code>Vxlan</code> \u548c <code>STT</code> \u4e09\u79cd\u96a7\u9053\u5c01\u88c5\u534f\u8bae\u3002 \u8fd9\u4e09\u79cd\u534f\u8bae\u5728\u529f\u80fd\uff0c\u6027\u80fd\u548c\u6613\u7528\u6027\u4e0a\u5b58\u5728\u7740\u533a\u522b\uff0c\u672c\u6587\u6863\u5c06\u4ecb\u7ecd\u4e09\u79cd\u534f\u8bae\u5728\u4f7f\u7528\u4e2d\u7684\u5dee\u5f02\uff0c\u7528\u6237\u53ef\u6839\u636e\u81ea\u5df1\u7684\u60c5\u51b5\u8fdb\u884c\u9009\u62e9\u3002</p>"},{"location":"reference/tunnel-protocol/#geneve","title":"Geneve","text":"<p><code>Geneve</code> \u534f\u8bae\u4e3a Kube-OVN \u90e8\u7f72\u65f6\u9009\u62e9\u7684\u9ed8\u8ba4\u96a7\u9053\u534f\u8bae\uff0c\u4e5f\u662f OVN \u9ed8\u8ba4\u63a8\u8350\u7684\u96a7\u9053\u534f\u8bae\u3002\u8be5\u534f\u8bae\u5728\u5185\u6838\u4e2d\u5f97\u5230\u4e86\u5e7f\u6cdb\u7684\u652f\u6301\uff0c \u5e76\u53ef\u4ee5\u5229\u7528\u73b0\u4ee3\u7f51\u5361\u7684\u901a\u7528 Offload \u80fd\u529b\u8fdb\u884c\u52a0\u901f\u3002\u7531\u4e8e <code>Geneve</code> \u6709\u7740\u53ef\u53d8\u957f\u7684\u5934\u90e8\uff0c\u53ef\u4ee5\u4f7f\u7528 24bit \u7a7a\u95f4\u6765\u6807\u5fd7\u4e0d\u540c\u7684 datapath \u7528\u6237\u53ef\u4ee5\u521b\u5efa\u66f4\u591a\u6570\u91cf\u7684\u865a\u62df\u7f51\u7edc\u3002</p> <p>\u5982\u679c\u4f7f\u7528 Mellanox \u6216\u82af\u542f\u6e90\u7684\u667a\u80fd\u7f51\u5361 OVS \u5378\u8f7d\uff0c<code>Geneve</code> \u9700\u8981\u8f83\u9ad8\u7248\u672c\u7684\u5185\u6838\u652f\u6301\uff0c\u9700\u8981\u9009\u62e9 5.4 \u4ee5\u4e0a\u7684\u4e0a\u6e38\u5185\u6838\uff0c \u6216 backport \u4e86\u8be5\u529f\u80fd\u7684\u5176\u4ed6\u517c\u5bb9\u5185\u6838\u3002</p> <p>\u7531\u4e8e\u4f7f\u7528 UDP \u8fdb\u884c\u5c01\u88c5\uff0c\u8be5\u534f\u8bae\u5728\u5904\u7406 TCP over UDP \u65f6\u4e0d\u80fd\u5f88\u597d\u7684\u5229\u7528\u73b0\u4ee3\u7f51\u5361\u7684 TCP \u76f8\u5173\u5378\u8f7d\uff0c\u5728\u5904\u7406\u5927\u5305\u65f6\u4f1a\u6d88\u8017\u8f83\u591a CPU \u8d44\u6e90\u3002</p>"},{"location":"reference/tunnel-protocol/#vxlan","title":"Vxlan","text":"<p><code>Vxlan</code> \u4e3a\u4e0a\u6e38 OVN \u8fd1\u671f\u652f\u6301\u7684\u534f\u8bae\uff0c\u8be5\u534f\u8bae\u5728\u5185\u6838\u4e2d\u5f97\u5230\u4e86\u5e7f\u6cdb\u7684\u652f\u6301\uff0c \u5e76\u53ef\u4ee5\u5229\u7528\u73b0\u4ee3\u7f51\u5361\u7684\u901a\u7528 Offload \u80fd\u529b\u8fdb\u884c\u52a0\u901f\u3002 \u7531\u4e8e\u8be5\u534f\u8bae\u5934\u90e8\u957f\u5ea6\u6709\u9650\uff0c\u5e76\u4e14 OVN \u9700\u8981\u4f7f\u7528\u989d\u5916\u7684\u7a7a\u95f4\u8fdb\u884c\u7f16\u6392\uff0cdatapath \u7684\u6570\u91cf\u5b58\u5728\u9650\u5236\uff0c\u6700\u591a\u53ea\u80fd\u521b\u5efa 4096 \u4e2a datapath\uff0c \u6bcf\u4e2a datapath \u4e0b\u6700\u591a 4096 \u4e2a\u7aef\u53e3\u3002\u540c\u65f6\u7531\u4e8e\u7a7a\u95f4\u6709\u9650\uff0c\u57fa\u4e8e <code>inport</code> \u7684 ACL \u6ca1\u6709\u8fdb\u884c\u652f\u6301\u3002</p> <p>\u5982\u679c\u4f7f\u7528 Mellanox \u6216\u82af\u542f\u6e90\u7684\u667a\u80fd\u7f51\u5361 OVS \u5378\u8f7d\uff0c<code>Vxlan</code> \u7684\u5378\u8f7d\u5728\u5e38\u89c1\u5185\u6838\u4e2d\u5df2\u83b7\u5f97\u652f\u6301\u3002</p> <p>\u7531\u4e8e\u4f7f\u7528 UDP \u8fdb\u884c\u5c01\u88c5\uff0c\u8be5\u534f\u8bae\u5728\u5904\u7406 TCP over UDP \u65f6\u4e0d\u80fd\u5f88\u597d\u7684\u5229\u7528\u73b0\u4ee3\u7f51\u5361\u7684 TCP \u76f8\u5173\u5378\u8f7d\uff0c\u5728\u5904\u7406\u5927\u5305\u65f6\u4f1a\u6d88\u8017\u8f83\u591a CPU \u8d44\u6e90\u3002</p>"},{"location":"reference/tunnel-protocol/#stt","title":"STT","text":"<p><code>STT</code> \u534f\u8bae\u4e3a OVN \u8f83\u65e9\u652f\u6301\u7684\u96a7\u9053\u534f\u8bae\uff0c\u8be5\u534f\u8bae\u4f7f\u7528\u7c7b TCP \u7684\u5934\u90e8\uff0c\u53ef\u4ee5\u5145\u5206\u5229\u7528\u73b0\u4ee3\u7f51\u5361\u901a\u7528\u7684 TCP \u5378\u8f7d\u80fd\u529b\uff0c\u5927\u5e45\u63d0\u5347 TCP \u7684\u541e\u5410\u91cf\u3002\u540c\u65f6\u8be5\u534f\u8bae\u5934\u90e8\u8f83\u957f\u53ef\u652f\u6301\u5b8c\u6574\u7684 OVN \u80fd\u529b\u548c\u5927\u89c4\u6a21\u7684 datapath\u3002</p> <p>\u8be5\u534f\u8bae\u672a\u5728\u5185\u6838\u4e2d\u652f\u6301\uff0c\u82e5\u8981\u4f7f\u7528\u9700\u8981\u989d\u5916\u7f16\u8bd1 OVS \u5185\u6838\u6a21\u5757\uff0c\u5e76\u5728\u5347\u7ea7\u5185\u6838\u65f6\u5bf9\u5e94\u518d\u6b21\u7f16\u8bd1\u65b0\u7248\u672c\u5185\u6838\u6a21\u5757\u3002</p> <p>\u8be5\u534f\u8bae\u76ee\u524d\u672a\u88ab\u667a\u80fd\u7f51\u5361\u652f\u6301\uff0c\u65e0\u6cd5\u4f7f\u7528 OVS \u7684\u5378\u8f7d\u80fd\u529b\u3002</p>"},{"location":"reference/tunnel-protocol/#_2","title":"\u53c2\u8003\u8d44\u6599","text":"<ul> <li>https://ipwithease.com/vxlan-vs-geneve-understand-the-difference/</li> <li>OVN FAQ</li> <li>What is Geneve</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"reference/underlay-topology/","title":"Underlay \u6d41\u91cf\u62d3\u6251","text":"<p>\u672c\u6587\u6863\u4ecb\u7ecd Underlay \u6a21\u5f0f\u4e0b\u6d41\u91cf\u5728\u4e0d\u540c\u60c5\u51b5\u4e0b\u7684\u8f6c\u53d1\u8def\u5f84\u3002</p>"},{"location":"reference/underlay-topology/#_1","title":"\u540c\u8282\u70b9\u540c\u5b50\u7f51","text":"<p>\u5185\u90e8\u903b\u8f91\u4ea4\u6362\u673a\u76f4\u63a5\u4ea4\u6362\u6570\u636e\u5305\uff0c\u4e0d\u8fdb\u5165\u5916\u90e8\u7f51\u7edc\u3002</p> <p></p>"},{"location":"reference/underlay-topology/#_2","title":"\u8de8\u8282\u70b9\u540c\u5b50\u7f51","text":"<p>\u6570\u636e\u5305\u7ecf\u7531\u8282\u70b9\u7f51\u5361\u8fdb\u5165\u5916\u90e8\u4ea4\u6362\u673a\uff0c\u7531\u5916\u90e8\u4ea4\u6362\u673a\u8fdb\u884c\u4ea4\u6362\u3002</p> <p></p>"},{"location":"reference/underlay-topology/#_3","title":"\u540c\u8282\u70b9\u4e0d\u540c\u5b50\u7f51","text":"<p>\u6570\u636e\u5305\u7ecf\u7531\u8282\u70b9\u7f51\u5361\u8fdb\u5165\u5916\u90e8\u7f51\u7edc\uff0c\u7531\u5916\u90e8\u4ea4\u6362\u673a\u53ca\u8def\u7531\u5668\u8fdb\u884c\u4ea4\u6362\u548c\u8def\u7531\u8f6c\u53d1\u3002</p> <p></p> <p>\u6b64\u5904 br-provider-1 \u548c br-provider-2 \u53ef\u4ee5\u662f\u540c\u4e00\u4e2a OVS \u7f51\u6865\uff0c\u5373\u591a\u4e2a\u4e0d\u540c\u5b50\u7f51\u53ef\u4ee5\u4f7f\u7528\u540c\u4e00\u4e2a Provider Network\u3002</p>"},{"location":"reference/underlay-topology/#_4","title":"\u8de8\u8282\u70b9\u4e0d\u540c\u5b50\u7f51","text":"<p>\u6570\u636e\u5305\u7ecf\u7531\u8282\u70b9\u7f51\u5361\u8fdb\u5165\u5916\u90e8\u7f51\u7edc\uff0c\u7531\u5916\u90e8\u4ea4\u6362\u673a\u53ca\u8def\u7531\u5668\u8fdb\u884c\u4ea4\u6362\u548c\u8def\u7531\u8f6c\u53d1\u3002</p> <p></p>"},{"location":"reference/underlay-topology/#_5","title":"\u8bbf\u95ee\u5916\u90e8","text":"<p>\u6570\u636e\u5305\u7ecf\u7531\u8282\u70b9\u7f51\u5361\u8fdb\u5165\u5916\u90e8\u7f51\u7edc\uff0c\u7531\u5916\u90e8\u4ea4\u6362\u673a\u53ca\u8def\u7531\u5668\u8fdb\u884c\u4ea4\u6362\u548c\u8def\u7531\u8f6c\u53d1\u3002</p> <p></p> <p>\u8282\u70b9\u4e0e Pod \u4e4b\u95f4\u7684\u901a\u4fe1\u5927\u4f53\u4e0a\u4e5f\u9075\u5faa\u6b64\u903b\u8f91\u3002</p>"},{"location":"reference/underlay-topology/#vlan-tag","title":"\u65e0 Vlan Tag \u4e0b\u603b\u89c8","text":""},{"location":"reference/underlay-topology/#vlan","title":"\u591a VLAN \u603b\u89c8","text":""},{"location":"reference/underlay-topology/#pod-service-ip","title":"Pod \u8bbf\u95ee Service IP","text":"<p>Kube-OVN \u4e3a\u6bcf\u4e2a Kubernetes Service \u5728\u6bcf\u4e2a\u5b50\u7f51\u7684\u903b\u8f91\u4ea4\u6362\u673a\u4e0a\u914d\u7f6e\u4e86\u8d1f\u8f7d\u5747\u8861\u3002 \u5f53 Pod \u901a\u8fc7\u8bbf\u95ee Service IP \u8bbf\u95ee\u5176\u5b83 Pod \u65f6\uff0c\u4f1a\u6784\u9020\u4e00\u4e2a\u76ee\u7684\u5730\u5740\u4e3a Service IP\u3001\u76ee\u7684 MAC \u5730\u5740\u4e3a\u7f51\u5173 MAC \u5730\u5740\u7684\u7f51\u7edc\u5305\u3002 \u7f51\u7edc\u5305\u8fdb\u5165\u903b\u8f91\u4ea4\u6362\u673a\u540e\uff0c\u8d1f\u8f7d\u5747\u8861\u4f1a\u5bf9\u7f51\u7edc\u5305\u8fdb\u884c\u62e6\u622a\u548c DNAT \u5904\u7406\uff0c\u5c06\u76ee\u7684 IP \u548c\u7aef\u53e3\u4fee\u6539\u4e3a Service \u5bf9\u5e94\u7684\u67d0\u4e2a Endpoint \u7684 IP \u548c\u7aef\u53e3\u3002 \u7531\u4e8e\u903b\u8f91\u4ea4\u6362\u673a\u5e76\u672a\u4fee\u6539\u7f51\u7edc\u5305\u7684\u4e8c\u5c42\u76ee\u7684 MAC \u5730\u5740\uff0c\u7f51\u7edc\u5305\u5728\u8fdb\u5165\u5916\u90e8\u4ea4\u6362\u673a\u540e\u4ecd\u7136\u4f1a\u9001\u5230\u5916\u90e8\u7f51\u5173\uff0c\u6b64\u65f6\u9700\u8981\u5916\u90e8\u7f51\u5173\u5bf9\u7f51\u7edc\u5305\u8fdb\u884c\u8f6c\u53d1\u3002</p>"},{"location":"reference/underlay-topology/#service-pod","title":"Service \u540e\u7aef\u4e3a\u540c\u8282\u70b9\u540c\u5b50\u7f51 Pod","text":""},{"location":"reference/underlay-topology/#service-pod_1","title":"Service \u540e\u7aef\u4e3a\u540c\u8282\u70b9\u4e0d\u540c\u5b50\u7f51 Pod","text":"<p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"start/one-step-install/","title":"\u4e00\u952e\u5b89\u88c5","text":"<p>Kube-OVN \u63d0\u4f9b\u4e86\u4e00\u952e\u5b89\u88c5\u811a\u672c\uff0c\u53ef\u4ee5\u5e2e\u52a9\u4f60\u5feb\u901f\u5b89\u88c5\u4e00\u4e2a\u9ad8\u53ef\u7528\uff0c\u751f\u4ea7\u5c31\u7eea\u7684 Kube-OVN \u5bb9\u5668\u7f51\u7edc\uff0c\u9ed8\u8ba4\u90e8\u7f72\u4e3a Overlay \u7c7b\u578b\u7f51\u7edc\u3002</p> <p>\u4ece Kube-OVN v1.12.0 \u7248\u672c\u5f00\u59cb\uff0c\u652f\u6301 Helm Chart \u5b89\u88c5\uff0c\u9ed8\u8ba4\u90e8\u7f72\u4e3a Overlay \u7c7b\u578b\u7f51\u7edc\u3002</p> <p>\u5982\u679c\u9ed8\u8ba4\u7f51\u7edc\u9700\u8981\u642d\u5efa Underlay/Vlan \u7f51\u7edc\uff0c\u8bf7\u53c2\u8003 Underlay \u7f51\u7edc\u652f\u6301\u3002</p> <p>\u5b89\u88c5\u524d\u8bf7\u53c2\u8003\u51c6\u5907\u5de5\u4f5c\u786e\u8ba4\u73af\u5883\u914d\u7f6e\u6b63\u786e\u3002</p>"},{"location":"start/one-step-install/#_2","title":"\u811a\u672c\u5b89\u88c5","text":""},{"location":"start/one-step-install/#_3","title":"\u4e0b\u8f7d\u5b89\u88c5\u811a\u672c","text":"<p>\u6211\u4eec\u63a8\u8350\u5728\u751f\u4ea7\u73af\u5883\u4f7f\u7528\u7a33\u5b9a\u7684 release \u7248\u672c\uff0c\u8bf7\u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u4e0b\u8f7d\u7a33\u5b9a\u7248\u672c\u5b89\u88c5\u811a\u672c\uff1a</p> <pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/images/install.sh\n</code></pre> <p>\u5982\u679c\u5bf9 master \u5206\u652f\u7684\u6700\u65b0\u529f\u80fd\u611f\u5174\u8da3\uff0c\u60f3\u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u4e0b\u8f7d\u5f00\u53d1\u7248\u672c\u90e8\u7f72\u811a\u672c\uff1a</p> <pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/images/install.sh\n</code></pre>"},{"location":"start/one-step-install/#_4","title":"\u4fee\u6539\u914d\u7f6e\u53c2\u6570","text":"<p>\u4f7f\u7528\u7f16\u8f91\u5668\u6253\u5f00\u811a\u672c\uff0c\u5e76\u4fee\u6539\u4e0b\u5217\u53d8\u91cf\u4e3a\u9884\u671f\u503c\uff1a</p> <pre><code>REGISTRY=\"kubeovn\"                     # \u955c\u50cf\u4ed3\u5e93\u5730\u5740\nVERSION=\"v1.13.0\"                      # \u955c\u50cf\u7248\u672c/Tag\nPOD_CIDR=\"10.16.0.0/16\"                # \u9ed8\u8ba4\u5b50\u7f51 CIDR \u4e0d\u8981\u548c SVC/NODE/JOIN CIDR \u91cd\u53e0\nSVC_CIDR=\"10.96.0.0/12\"                # \u9700\u8981\u548c apiserver \u7684 service-cluster-ip-range \u4fdd\u6301\u4e00\u81f4\nJOIN_CIDR=\"100.64.0.0/16\"              # Pod \u548c\u4e3b\u673a\u901a\u4fe1\u7f51\u7edc CIDR\uff0c\u4e0d\u8981\u548c SVC/NODE/POD CIDR \u91cd\u53e0 \nLABEL=\"node-role.kubernetes.io/master\" # \u90e8\u7f72 OVN DB \u8282\u70b9\u7684\u6807\u7b7e\nIFACE=\"\"                               # \u5bb9\u5668\u7f51\u7edc\u6240\u4f7f\u7528\u7684\u7684\u5bbf\u4e3b\u673a\u7f51\u5361\u540d\uff0c\u5982\u679c\u4e3a\u7a7a\u5219\u4f7f\u7528 Kubernetes \u4e2d\u7684 Node IP \u6240\u5728\u7f51\u5361\nTUNNEL_TYPE=\"geneve\"                   # \u96a7\u9053\u5c01\u88c5\u534f\u8bae\uff0c\u53ef\u9009 geneve, vxlan \u6216 stt\uff0cstt \u9700\u8981\u5355\u72ec\u7f16\u8bd1 ovs \u5185\u6838\u6a21\u5757\n</code></pre> <p>\u53ef\u4f7f\u7528\u6b63\u5219\u8868\u8fbe\u5f0f\u6765\u5339\u914d\u7f51\u5361\u540d\uff0c\u4f8b\u5982 <code>IFACE=enp6s0f0,eth.*</code>\u3002</p>"},{"location":"start/one-step-install/#_5","title":"\u6267\u884c\u5b89\u88c5\u811a\u672c","text":"<p><code>bash install.sh</code></p> <p>\u7b49\u5f85\u5b89\u88c5\u5b8c\u6210\u3002</p>"},{"location":"start/one-step-install/#helm-chart","title":"Helm Chart \u5b89\u88c5","text":"<p>\u7531\u4e8e Kube-OVN \u7684\u5b89\u88c5\uff0c\u9700\u8981\u8bbe\u7f6e\u4e00\u4e9b\u53c2\u6570\uff0c\u56e0\u6b64\u4f7f\u7528 Helm \u5b89\u88c5 Kube-OVN\uff0c\u9700\u8981\u6309\u7167\u4ee5\u4e0b\u6b65\u9aa4\u6267\u884c\u3002</p>"},{"location":"start/one-step-install/#ip","title":"\u67e5\u770b\u8282\u70b9 IP \u5730\u5740","text":"<pre><code>$ kubectl get node -o wide\nNAME                     STATUS     ROLES           AGE   VERSION   INTERNAL-IP   EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION      CONTAINER-RUNTIME\nkube-ovn-control-plane   NotReady   control-plane   20h   v1.26.0   172.18.0.3    &lt;none&gt;        Ubuntu 22.04.1 LTS   5.10.104-linuxkit   containerd://1.6.9\nkube-ovn-worker          NotReady   &lt;none&gt;          20h   v1.26.0   172.18.0.2    &lt;none&gt;        Ubuntu 22.04.1 LTS   5.10.104-linuxkit   containerd://1.6.9\n</code></pre>"},{"location":"start/one-step-install/#master","title":"\u53bb\u6389\u96c6\u7fa4 master \u8282\u70b9\u6c61\u70b9","text":"<pre><code>$ kubectl taint node kube-ovn-control-plane node-role.kubernetes.io/control-plane:NoSchedule-\nnode/kube-ovn-control-plane untainted\n</code></pre> <p>\u5982\u679c\u786e\u5b9a\u4e0d\u9700\u8981\u5728 master \u8282\u70b9\u8c03\u5ea6\u4e1a\u52a1 Pod\uff0c\u8fd9\u4e00\u6b65\u53ef\u4ee5\u8df3\u8fc7\u3002</p>"},{"location":"start/one-step-install/#label","title":"\u7ed9\u8282\u70b9\u6dfb\u52a0 label","text":"<pre><code>$ kubectl label node -lbeta.kubernetes.io/os=linux kubernetes.io/os=linux --overwrite\nnode/kube-ovn-control-plane not labeled\nnode/kube-ovn-worker not labeled\n\n$ kubectl label node -lnode-role.kubernetes.io/control-plane kube-ovn/role=master --overwrite\nnode/kube-ovn-control-plane labeled\n\n# \u4ee5\u4e0b label \u7528\u4e8e dpdk \u955c\u50cf\u7684\u5b89\u88c5\uff0c\u975e dpdk \u60c5\u51b5\uff0c\u53ef\u4ee5\u5ffd\u7565\n$ kubectl label node -lovn.kubernetes.io/ovs_dp_type!=userspace ovn.kubernetes.io/ovs_dp_type=kernel --overwrite\nnode/kube-ovn-control-plane labeled\nnode/kube-ovn-worker labeled\n</code></pre>"},{"location":"start/one-step-install/#helm-repo","title":"\u6dfb\u52a0 Helm Repo \u4fe1\u606f","text":"<pre><code>$ helm repo add kubeovn https://kubeovn.github.io/kube-ovn/\n\"kubeovn\" has been added to your repositories\n\n$ helm repo list\nNAME            URL\nkubeovn         https://kubeovn.github.io/kube-ovn/\n\n$ helm search repo kubeovn\nNAME                CHART VERSION   APP VERSION DESCRIPTION\nkubeovn/kube-ovn    0.1.0           1.12.0      Helm chart for Kube-OVN\n</code></pre>"},{"location":"start/one-step-install/#helm-install-kube-ovn","title":"\u6267\u884c helm install \u5b89\u88c5 Kube-OVN","text":"<p>Node0IP\u3001Node1IP\u3001Node2IP \u53c2\u6570\u5206\u522b\u4e3a\u96c6\u7fa4 master \u8282\u70b9\u7684 IP \u5730\u5740\u3002\u5176\u4ed6\u53c2\u6570\u7684\u8bbe\u7f6e\uff0c\u53ef\u4ee5\u53c2\u8003 values.yaml \u6587\u4ef6\u4e2d\u53d8\u91cf\u5b9a\u4e49\u3002</p> <pre><code># \u5355 master \u8282\u70b9\u73af\u5883\u5b89\u88c5\n$ helm install kube-ovn kubeovn/kube-ovn --set MASTER_NODES=${Node0IP}\n\n# \u4ee5\u4e0a\u8fb9\u7684 node \u4fe1\u606f\u4e3a\u4f8b\uff0c\u6267\u884c\u5b89\u88c5\u547d\u4ee4\n$ helm install kube-ovn kubeovn/kube-ovn --set MASTER_NODES=172.18.0.3\nNAME: kube-ovn\nLAST DEPLOYED: Fri Mar 31 12:43:43 2023\nNAMESPACE: default\nSTATUS: deployed\nREVISION: 1\nTEST SUITE: None\n\n# \u9ad8\u53ef\u7528\u96c6\u7fa4\u5b89\u88c5\n$ helm install kube-ovn kubeovn/kube-ovn --set MASTER_NODES=${Node0IP},${Node1IP},${Node2IP}, --set replicaCount=3\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"start/prepare/","title":"\u51c6\u5907\u5de5\u4f5c","text":"<p>Kube-OVN \u662f\u4e00\u4e2a\u7b26\u5408 CNI \u89c4\u8303\u7684\u7f51\u7edc\u7ec4\u4ef6\uff0c\u5176\u8fd0\u884c\u9700\u8981\u4f9d\u8d56 Kubernetes \u73af\u5883\u53ca\u5bf9\u5e94\u7684\u5185\u6838\u7f51\u7edc\u6a21\u5757\u3002 \u4ee5\u4e0b\u662f\u901a\u8fc7\u6d4b\u8bd5\u7684\u64cd\u4f5c\u7cfb\u7edf\u548c\u8f6f\u4ef6\u7248\u672c\uff0c\u73af\u5883\u914d\u7f6e\u548c\u6240\u9700\u8981\u5f00\u653e\u7684\u7aef\u53e3\u4fe1\u606f\u3002</p>"},{"location":"start/prepare/#_2","title":"\u8f6f\u4ef6\u7248\u672c","text":"<ul> <li>Kubernetes &gt;= 1.23\u3002</li> <li>Docker &gt;= 1.12.6, Containerd &gt;= 1.3.4\u3002</li> <li>\u64cd\u4f5c\u7cfb\u7edf: CentOS 7/8, Ubuntu 16.04/18.04/20.04\u3002</li> <li>\u5176\u4ed6 Linux \u53d1\u884c\u7248\uff0c\u9700\u8981\u68c0\u67e5\u4e00\u4e0b\u5185\u6838\u6a21\u5757\u662f\u5426\u5b58\u5728 <code>geneve</code>, <code>openvswitch</code>, <code>ip_tables</code> \u548c <code>iptable_nat</code>\uff0cKube-OVN \u6b63\u5e38\u5de5\u4f5c\u4f9d\u8d56\u4e0a\u8ff0\u6a21\u5757\u3002</li> </ul> <p>\u6ce8\u610f\u4e8b\u9879\uff1a</p> <ol> <li>\u5982\u679c\u5185\u6838\u7248\u672c\u4e3a 3.10.0-862 \u5185\u6838 <code>netfilter</code> \u6a21\u5757\u5b58\u5728 bug \u4f1a\u5bfc\u81f4 Kube-OVN \u5185\u7f6e\u8d1f\u8f7d\u5747\u8861\u5668\u65e0\u6cd5\u5de5\u4f5c\uff0c\u9700\u8981\u5bf9\u5185\u6838\u5347\u7ea7\uff0c\u5efa\u8bae\u4f7f\u7528 CentOS \u5b98\u65b9\u5bf9\u5e94\u7248\u672c\u6700\u65b0\u5185\u6838\u4fdd\u8bc1\u7cfb\u7edf\u7684\u5b89\u5168\u3002\u76f8\u5173\u5185\u6838 bug \u53c2\u8003 Floating IPs broken after kernel upgrade to Centos/RHEL 7.5 - DNAT not working\u3002</li> <li>Rocky Linux 8.6 \u7684\u5185\u6838 4.18.0-372.9.1.el8.x86_64 \u5b58\u5728 TCP \u901a\u4fe1\u95ee\u9898 TCP connection failed in Rocky Linux 8.6\uff0c\u8bf7\u5347\u7ea7\u5185\u6838\u81f3 4.18.0-372.13.1.el8_6.x86_64 \u6216\u66f4\u9ad8\u7248\u672c\u3002</li> <li>\u5982\u679c\u5185\u6838\u7248\u672c\u4e3a 4.4 \u5219\u5bf9\u5e94\u7684\u5185\u6838 <code>openvswitch</code> \u6a21\u5757\u5b58\u5728\u95ee\u9898\uff0c\u5efa\u8bae\u5347\u7ea7\u6216\u624b\u52a8\u7f16\u8bd1 <code>openvswitch</code> \u65b0\u7248\u672c\u6a21\u5757\u8fdb\u884c\u66f4\u65b0</li> <li>Geneve \u96a7\u9053\u5efa\u7acb\u9700\u8981\u68c0\u67e5 IPv6\uff0c\u53ef\u901a\u8fc7 <code>cat /proc/cmdline</code> \u68c0\u67e5\u5185\u6838\u542f\u52a8\u53c2\u6570\uff0c \u76f8\u5173\u5185\u6838 bug \u8bf7\u53c2\u8003 Geneve tunnels don't work when ipv6 is disabled\u3002</li> </ol>"},{"location":"start/prepare/#_3","title":"\u73af\u5883\u914d\u7f6e","text":"<ul> <li>Kernel \u542f\u52a8\u9700\u8981\u5f00\u542f ipv6, \u5982\u679c kernel \u542f\u52a8\u53c2\u6570\u5305\u542b <code>ipv6.disable=1</code> \u9700\u8981\u5c06\u5176\u8bbe\u7f6e\u4e3a 0\u3002</li> <li><code>kube-proxy</code> \u6b63\u5e38\u5de5\u4f5c\uff0cKube-OVN \u53ef\u4ee5\u901a\u8fc7 SVC IP \u8bbf\u95ee\u5230 <code>kube-apiserver</code>\u3002</li> <li>\u786e\u8ba4 kubelet \u914d\u7f6e\u53c2\u6570\u5f00\u542f\u4e86 CNI\uff0c\u5e76\u4e14\u914d\u7f6e\u5728\u6807\u51c6\u8def\u5f84\u4e0b, kubelet \u542f\u52a8\u65f6\u5e94\u5305\u542b\u5982\u4e0b\u53c2\u6570 <code>--network-plugin=cni --cni-bin-dir=/opt/cni/bin --cni-conf-dir=/etc/cni/net.d</code>\u3002</li> <li>\u786e\u8ba4\u672a\u5b89\u88c5\u5176\u4ed6\u7f51\u7edc\u63d2\u4ef6\uff0c\u6216\u8005\u5176\u4ed6\u7f51\u7edc\u63d2\u4ef6\u5df2\u7ecf\u88ab\u6e05\u9664\uff0c\u68c0\u67e5 <code>/etc/cni/net.d/</code> \u8def\u5f84\u4e0b\u65e0\u5176\u4ed6\u7f51\u7edc\u63d2\u4ef6\u914d\u7f6e\u6587\u4ef6\u3002\u5982\u679c\u4e4b\u524d\u5b89\u88c5\u8fc7\u5176\u4ed6\u7f51\u7edc\u63d2\u4ef6\uff0c\u5efa\u8bae\u5220\u9664\u540e\u91cd\u542f\u673a\u5668\u6e05\u7406\u6b8b\u7559\u7f51\u7edc\u8d44\u6e90\u3002</li> </ul>"},{"location":"start/prepare/#_4","title":"\u7aef\u53e3\u4fe1\u606f","text":"\u7ec4\u4ef6 \u7aef\u53e3 \u7528\u9014 ovn-central 6641/tcp, 6642/tcp, 6643/tcp, 6644/tcp ovn-db \u548c raft server \u76d1\u542c\u7aef\u53e3 ovs-ovn Geneve 6081/udp, STT 7471/tcp, Vxlan 4789/udp \u96a7\u9053\u7aef\u53e3 kube-ovn-controller 10660/tcp \u76d1\u63a7\u76d1\u542c\u7aef\u53e3 kube-ovn-daemon 10665/tcp \u76d1\u63a7\u76d1\u542c\u7aef\u53e3 kube-ovn-monitor 10661/tcp \u76d1\u63a7\u76d1\u542c\u7aef\u53e3 <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"start/sealos-install/","title":"\u4f7f\u7528 sealos \u4e00\u952e\u90e8\u7f72 Kubernetes \u548c Kube-OVN","text":"<p>sealos \u4f5c\u4e3a Kubernetes \u7684\u4e00\u4e2a\u53d1\u884c\u7248\uff0c\u901a\u8fc7\u6781\u7b80\u7684\u4f7f\u7528\u65b9\u5f0f\u548c\u56fd\u5185\u7684\u955c\u50cf\u4ed3\u5e93\uff0c\u53ef\u4ee5\u5e2e\u52a9\u7528\u6237\u5feb\u901f\u4ece\u96f6\u521d\u59cb\u5316\u4e00\u4e2a\u5bb9\u5668\u96c6\u7fa4\u3002 \u901a\u8fc7\u4f7f\u7528 sealos \u7528\u6237\u53ef\u4ee5\u901a\u8fc7\u4e00\u6761\u547d\u4ee4\u5728\u51e0\u5206\u949f\u5185\u90e8\u7f72\u51fa\u4e00\u4e2a\u5b89\u88c5\u597d Kube-OVN \u7684 Kubernetes \u96c6\u7fa4\u3002</p>"},{"location":"start/sealos-install/#sealos","title":"\u4e0b\u8f7d\u5b89\u88c5 sealos","text":"AMD64 ARM64 <pre><code>wget  https://github.com/labring/sealos/releases/download/v4.1.4/sealos_4.1.4_linux_amd64.tar.gz  &amp;&amp; \\\ntar -zxvf sealos_4.1.4_linux_amd64.tar.gz sealos &amp;&amp;  chmod +x sealos &amp;&amp; mv sealos /usr/bin\n</code></pre> <pre><code>wget  https://github.com/labring/sealos/releases/download/v4.1.4/sealos_4.1.4_linux_arm64.tar.gz  &amp;&amp; \\\ntar -zxvf sealos_4.1.4_linux_arm64.tar.gz sealos &amp;&amp;  chmod +x sealos &amp;&amp; mv sealos /usr/bin\n</code></pre>"},{"location":"start/sealos-install/#kubernetes-kube-ovn","title":"\u90e8\u7f72 Kubernetes \u548c Kube-OVN","text":"<pre><code>```bash\nsealos run labring/kubernetes:v1.24.3 labring/kube-ovn:v1.10.5 \\\n  --masters [masters ips seperated by comma] \\\n  --nodes [nodes ips seperated by comma] -p [your-ssh-passwd]\n```\n</code></pre>"},{"location":"start/sealos-install/#_1","title":"\u7b49\u5f85\u90e8\u7f72\u5b8c\u6210","text":"<pre><code>```bash\n[Step 6/6] Finish\n\n                    ,,,,\n                    ,::,\n                   ,,::,,,,\n            ,,,,,::::::::::::,,,,,\n         ,,,::::::::::::::::::::::,,,\n       ,,::::::::::::::::::::::::::::,,\n     ,,::::::::::::::::::::::::::::::::,,\n    ,::::::::::::::::::::::::::::::::::::,\n   ,:::::::::::::,,   ,,:::::,,,::::::::::,\n ,,:::::::::::::,       ,::,     ,:::::::::,\n ,:::::::::::::,   :x,  ,::  :,   ,:::::::::,\n,:::::::::::::::,  ,,,  ,::, ,,  ,::::::::::,\n,:::::::::::::::::,,,,,,:::::,,,,::::::::::::,    ,:,   ,:,            ,xx,                            ,:::::,   ,:,     ,:: :::,    ,x\n,::::::::::::::::::::::::::::::::::::::::::::,    :x: ,:xx:        ,   :xx,                          :xxxxxxxxx, :xx,   ,xx:,xxxx,   :x\n,::::::::::::::::::::::::::::::::::::::::::::,    :xxxxx:,  ,xx,  :x:  :xxx:x::,  ::xxxx:           :xx:,  ,:xxx  :xx, ,xx: ,xxxxx:, :x\n,::::::::::::::::::::::::::::::::::::::::::::,    :xxxxx,   :xx,  :x:  :xxx,,:xx,:xx:,:xx, ,,,,,,,,,xxx,    ,xx:   :xx:xx:  ,xxx,:xx::x\n,::::::,,::::::::,,::::::::,,:::::::,,,::::::,    :x:,xxx:  ,xx,  :xx  :xx:  ,xx,xxxxxx:, ,xxxxxxx:,xxx:,  ,xxx,    :xxx:   ,xxx, :xxxx\n,::::,    ,::::,   ,:::::,   ,,::::,    ,::::,    :x:  ,:xx,,:xx::xxxx,,xxx::xx: :xx::::x: ,,,,,,   ,xxxxxxxxx,     ,xx:    ,xxx,  :xxx\n,::::,    ,::::,    ,::::,    ,::::,    ,::::,    ,:,    ,:,  ,,::,,:,  ,::::,,   ,:::::,            ,,:::::,        ,,      :x:    ,::\n,::::,    ,::::,    ,::::,    ,::::,    ,::::,\n ,,,,,    ,::::,    ,::::,    ,::::,    ,:::,             ,,,,,,,,,,,,,\n          ,::::,    ,::::,    ,::::,    ,:::,        ,,,:::::::::::::::,\n          ,::::,    ,::::,    ,::::,    ,::::,  ,,,,:::::::::,,,,,,,:::,\n          ,::::,    ,::::,    ,::::,     ,::::::::::::,,,,,\n           ,,,,     ,::::,     ,,,,       ,,,::::,,,,\n                    ,::::,\n                    ,,::,\n\nThanks for choosing Kube-OVN!\nFor more advanced features, please read https://github.com/kubeovn/kube-ovn#documents\nIf you have any question, please file an issue https://github.com/kubeovn/kube-ovn/issues/new/choose\n2022-08-10T16:31:34 info succeeded in creating a new cluster, enjoy it!\n2022-08-10T16:31:34 info\n      ___           ___           ___           ___       ___           ___\n     /\\  \\         /\\  \\         /\\  \\         /\\__\\     /\\  \\         /\\  \\\n    /::\\  \\       /::\\  \\       /::\\  \\       /:/  /    /::\\  \\       /::\\  \\\n   /:/\\ \\  \\     /:/\\:\\  \\     /:/\\:\\  \\     /:/  /    /:/\\:\\  \\     /:/\\ \\  \\\n  _\\:\\~\\ \\  \\   /::\\~\\:\\  \\   /::\\~\\:\\  \\   /:/  /    /:/  \\:\\  \\   _\\:\\~\\ \\  \\\n /\\ \\:\\ \\ \\__\\ /:/\\:\\ \\:\\__\\ /:/\\:\\ \\:\\__\\ /:/__/    /:/__/ \\:\\__\\ /\\ \\:\\ \\ \\__\\\n \\:\\ \\:\\ \\/__/ \\:\\~\\:\\ \\/__/ \\/__\\:\\/:/  / \\:\\  \\    \\:\\  \\ /:/  / \\:\\ \\:\\ \\/__/\n  \\:\\ \\:\\__\\    \\:\\ \\:\\__\\        \\::/  /   \\:\\  \\    \\:\\  /:/  /   \\:\\ \\:\\__\\\n   \\:\\/:/  /     \\:\\ \\/__/        /:/  /     \\:\\  \\    \\:\\/:/  /     \\:\\/:/  /\n    \\::/  /       \\:\\__\\         /:/  /       \\:\\__\\    \\::/  /       \\::/  /\n     \\/__/         \\/__/         \\/__/         \\/__/     \\/__/         \\/__/\n\n                  Website :https://www.sealos.io/\n                  Address :github.com/labring/sealos\n```\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"start/underlay/","title":"Underlay \u7f51\u7edc\u5b89\u88c5","text":"<p>\u9ed8\u8ba4\u60c5\u51b5\u4e0b Kube-OVN \u7684\u9ed8\u8ba4\u5b50\u7f51\u4f7f\u7528 Geneve \u5bf9\u8de8\u4e3b\u673a\u6d41\u91cf\u8fdb\u884c\u5c01\u88c5\uff0c\u5728\u57fa\u7840\u8bbe\u65bd\u4e4b\u4e0a\u62bd\u8c61\u51fa\u4e00\u5c42\u865a\u62df\u7684 Overlay \u7f51\u7edc\u3002</p> <p>\u5bf9\u4e8e\u5e0c\u671b\u5bb9\u5668\u7f51\u7edc\u76f4\u63a5\u4f7f\u7528\u7269\u7406\u7f51\u7edc\u5730\u5740\u6bb5\u60c5\u51b5\uff0c\u53ef\u4ee5\u5c06 Kube-OVN \u7684\u9ed8\u8ba4\u5b50\u7f51\u5de5\u4f5c\u5728 Underlay \u6a21\u5f0f\uff0c\u53ef\u4ee5\u76f4\u63a5\u7ed9\u5bb9\u5668\u5206\u914d\u7269\u7406\u7f51\u7edc\u4e2d\u7684\u5730\u5740\u8d44\u6e90\uff0c\u8fbe\u5230\u66f4\u597d\u7684\u6027\u80fd\u4ee5\u53ca\u548c\u7269\u7406\u7f51\u7edc\u7684\u8fde\u901a\u6027\u3002</p> <p></p>"},{"location":"start/underlay/#_1","title":"\u529f\u80fd\u9650\u5236","text":"<p>\u7531\u4e8e\u8be5\u6a21\u5f0f\u4e0b\u5bb9\u5668\u7f51\u7edc\u76f4\u63a5\u4f7f\u7528\u7269\u7406\u7f51\u7edc\u8fdb\u884c\u4e8c\u5c42\u5305\u8f6c\u53d1\uff0cOverlay \u6a21\u5f0f\u4e0b\u7684 SNAT/EIP\uff0c \u5206\u5e03\u5f0f\u7f51\u5173/\u96c6\u4e2d\u5f0f\u7f51\u5173\u7b49 L3 \u529f\u80fd\u65e0\u6cd5\u4f7f\u7528\uff0cVPC \u7ea7\u522b\u7684\u9694\u79bb\u4e5f\u65e0\u6cd5\u5bf9 Underlay \u5b50\u7f51\u751f\u6548\u3002</p>"},{"location":"start/underlay/#macvlan","title":"\u548c Macvlan \u6bd4\u8f83","text":"<p>Kube-OVN \u7684 Underlay \u6a21\u5f0f\u548c Macvlan \u5de5\u4f5c\u6a21\u5f0f\u5341\u5206\u7c7b\u4f3c\uff0c\u5728\u529f\u80fd\u548c\u6027\u80fd\u4e0a\u4e3b\u8981\u6709\u4ee5\u4e0b\u51e0\u4e2a\u533a\u522b\uff1a</p> <ol> <li>\u7531\u4e8e Macvlan \u7684\u5185\u6838\u8def\u5f84\u66f4\u77ed\uff0c\u5e76\u4e14\u4e0d\u9700\u8981 OVS \u5bf9\u6570\u636e\u5305\u8fdb\u884c\u5904\u7406\uff0cMacvlan \u5728\u541e\u5410\u91cf\u548c\u5ef6\u8fdf\u6027\u80fd\u6307\u6807\u4e0a\u8868\u73b0\u4f1a\u66f4\u597d\u3002</li> <li>Kube-OVN \u901a\u8fc7\u6d41\u8868\u63d0\u4f9b\u4e86 arp-proxy \u529f\u80fd\uff0c\u53ef\u4ee5\u7f13\u89e3\u5927\u89c4\u6a21\u7f51\u7edc\u4e0b\u7684 arp \u5e7f\u64ad\u98ce\u66b4\u98ce\u9669\u3002</li> <li>\u7531\u4e8e Macvlan \u5de5\u4f5c\u5728\u5185\u6838\u5e95\u5c42\uff0c\u4f1a\u7ed5\u8fc7\u5bbf\u4e3b\u673a\u7684 netfilter\uff0cService \u548c NetworkPolicy \u529f\u80fd\u9700\u8981\u989d\u5916\u5f00\u53d1\u3002Kube-OVN \u901a\u8fc7 OVS \u6d41\u8868\u63d0\u4f9b\u4e86 Service \u548c NetworkPolicy \u7684\u80fd\u529b\u3002</li> <li>Kube-OVN \u7684 Underlay \u6a21\u5f0f\u76f8\u6bd4 Macvlan \u989d\u5916\u63d0\u4f9b\u4e86\u5730\u5740\u7ba1\u7406\uff0c\u56fa\u5b9a IP \u548c QoS \u7b49\u529f\u80fd\u3002</li> </ol>"},{"location":"start/underlay/#_2","title":"\u73af\u5883\u8981\u6c42","text":"<p>\u5728 Underlay \u6a21\u5f0f\u4e0b\uff0cOVS \u5c06\u4f1a\u6865\u63a5\u4e00\u4e2a\u8282\u70b9\u7f51\u5361\u5230 OVS \u7f51\u6865\uff0c\u5e76\u5c06\u6570\u636e\u5305\u76f4\u63a5\u901a\u8fc7\u8be5\u8282\u70b9\u7f51\u5361\u5bf9\u5916\u53d1\u9001\uff0cL2/L3 \u5c42\u9762\u7684\u8f6c\u53d1\u80fd\u529b\u9700\u8981\u4f9d\u8d56\u5e95\u5c42\u7f51\u7edc\u8bbe\u5907\u3002 \u9700\u8981\u9884\u5148\u5728\u5e95\u5c42\u7f51\u7edc\u8bbe\u5907\u914d\u7f6e\u5bf9\u5e94\u7684\u7f51\u5173\u3001Vlan \u548c\u5b89\u5168\u7b56\u7565\u7b49\u914d\u7f6e\u3002</p> <ol> <li>\u5bf9\u4e8e OpenStack \u7684 VM \u73af\u5883\uff0c\u9700\u8981\u5c06\u5bf9\u5e94\u7f51\u7edc\u7aef\u53e3\u7684 <code>PortSecurity</code> \u5173\u95ed\u3002</li> <li>\u5bf9\u4e8e VMware \u7684 vSwitch \u7f51\u7edc\uff0c\u9700\u8981\u5c06 <code>MAC Address Changes</code>, <code>Forged Transmits</code> \u548c <code>Promiscuous Mode Operation</code> \u8bbe\u7f6e\u4e3a <code>allow</code>\u3002</li> <li>\u5bf9\u4e8e Hyper-V \u865a\u62df\u5316\uff0c\u9700\u8981\u5f00\u542f\u865a\u62df\u673a\u7f51\u5361\u9ad8\u7ea7\u529f\u80fd\u4e2d\u7684 <code>MAC Address Spoofing</code>\u3002</li> <li>\u516c\u6709\u4e91\uff0c\u4f8b\u5982 AWS\u3001GCE\u3001\u963f\u91cc\u4e91\u7b49\u7531\u4e8e\u4e0d\u652f\u6301\u7528\u6237\u81ea\u5b9a\u4e49 Mac \u65e0\u6cd5\u652f\u6301 Underlay \u6a21\u5f0f\u7f51\u7edc\uff0c\u5728\u8fd9\u79cd\u573a\u666f\u4e0b\u5982\u679c\u60f3\u4f7f\u7528 Underlay \u63a8\u8350\u4f7f\u7528\u5bf9\u5e94\u516c\u6709\u4e91\u5382\u5546\u63d0\u4f9b\u7684 VPC-CNI\u3002</li> <li>\u6865\u63a5\u7f51\u5361\u4e0d\u80fd\u4e3a Linux Bridge\u3002</li> </ol> <p>\u5bf9\u4e8e\u7ba1\u7406\u7f51\u548c\u5bb9\u5668\u7f51\u4f7f\u7528\u540c\u4e00\u4e2a\u7f51\u5361\u7684\u60c5\u51b5\u4e0b\uff0cKube-OVN \u4f1a\u5c06\u7f51\u5361\u7684 Mac \u5730\u5740\u3001IP \u5730\u5740\u3001\u8def\u7531\u4ee5\u53ca MTU \u5c06\u8f6c\u79fb\u6216\u590d\u5236\u81f3\u5bf9\u5e94\u7684 OVS Bridge\uff0c \u4ee5\u652f\u6301\u5355\u7f51\u5361\u90e8\u7f72 Underlay \u7f51\u7edc\u3002OVS Bridge \u540d\u79f0\u683c\u5f0f\u4e3a <code>br-PROVIDER_NAME</code>\uff0c<code>PROVIDER_NAME</code> \u4e3a Provider \u7f51\u7edc\u540d\u79f0\uff08\u9ed8\u8ba4\u4e3a provider\uff09\u3002</p>"},{"location":"start/underlay/#_3","title":"\u90e8\u7f72\u65f6\u6307\u5b9a\u7f51\u7edc\u6a21\u5f0f","text":"<p>\u8be5\u90e8\u7f72\u6a21\u5f0f\u5c06\u9ed8\u8ba4\u5b50\u7f51\u8bbe\u7f6e\u4e3a Underlay \u6a21\u5f0f\uff0c\u6240\u6709\u672a\u6307\u5b9a\u5b50\u7f51\u7684 Pod \u5747\u4f1a\u9ed8\u8ba4\u8fd0\u884c\u5728 Underlay \u7f51\u7edc\u4e2d\u3002</p>"},{"location":"start/underlay/#_4","title":"\u4e0b\u8f7d\u5b89\u88c5\u811a\u672c","text":"<pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/images/install.sh\n</code></pre>"},{"location":"start/underlay/#_5","title":"\u4fee\u6539\u811a\u672c\u4e2d\u76f8\u5e94\u914d\u7f6e","text":"<pre><code>ENABLE_ARP_DETECT_IP_CONFLICT # \u5982\u6709\u9700\u8981\uff0c\u53ef\u4ee5\u9009\u62e9\u5173\u95ed vlan \u7f51\u7edc arp \u51b2\u7a81\u68c0\u6d4b\nNETWORK_TYPE                  # \u8bbe\u7f6e\u4e3a vlan\nVLAN_INTERFACE_NAME           # \u8bbe\u7f6e\u4e3a\u5bbf\u4e3b\u673a\u4e0a\u627f\u62c5\u5bb9\u5668\u6d41\u91cf\u7684\u7f51\u5361\uff0c\u4f8b\u5982 eth1\nVLAN_ID                       # \u4ea4\u6362\u673a\u6240\u63a5\u53d7\u7684 VLAN Tag\uff0c\u82e5\u8bbe\u7f6e\u4e3a 0 \u5219\u4e0d\u505a VLAN \u5c01\u88c5\nPOD_CIDR                      # \u8bbe\u7f6e\u4e3a\u7269\u7406\u7f51\u7edc CIDR\uff0c \u4f8b\u5982 192.168.1.0/24\nPOD_GATEWAY                   # \u8bbe\u7f6e\u4e3a\u7269\u7406\u7f51\u7edc\u7f51\u5173\uff0c\u4f8b\u5982 192.168.1.1\nEXCLUDE_IPS                   # \u6392\u9664\u8303\u56f4\uff0c\u907f\u514d\u5bb9\u5668\u7f51\u6bb5\u548c\u7269\u7406\u7f51\u7edc\u5df2\u7528 IP \u51b2\u7a81\uff0c\u4f8b\u5982 192.168.1.1..192.168.1.100\nENABLE_LB                     # \u5982\u679c Underlay \u5b50\u7f51\u9700\u8981\u4f7f\u7528 Service \u9700\u8981\u8bbe\u7f6e\u4e3a true \nEXCHANGE_LINK_NAME            # \u662f\u5426\u4ea4\u6362\u9ed8\u8ba4 provider-network \u4e0b OVS \u7f51\u6865\u548c\u6865\u63a5\u7f51\u5361\u7684\u540d\u5b57\uff0c\u9ed8\u8ba4\u4e3a false\nLS_DNAT_MOD_DL_DST            # DNAT \u65f6\u662f\u5426\u5bf9 MAC \u5730\u5740\u8fdb\u884c\u8f6c\u6362\uff0c\u53ef\u52a0\u901f Service \u7684\u8bbf\u95ee\uff0c\u9ed8\u8ba4\u4e3a true\n</code></pre>"},{"location":"start/underlay/#_6","title":"\u8fd0\u884c\u5b89\u88c5\u811a\u672c","text":"<pre><code>bash install.sh\n</code></pre>"},{"location":"start/underlay/#crd-underlay","title":"\u901a\u8fc7 CRD \u52a8\u6001\u521b\u5efa Underlay \u7f51\u7edc","text":"<p>\u8be5\u65b9\u5f0f\u53ef\u5728\u5b89\u88c5\u540e\u52a8\u6001\u7684\u521b\u5efa\u67d0\u4e2a Underlay \u5b50\u7f51\u4f9b Pod \u4f7f\u7528\u3002\u9700\u8981\u914d\u7f6e <code>ProviderNetwork</code>\uff0c<code>Vlan</code> \u548c <code>Subnet</code> \u4e09\u79cd\u81ea\u5b9a\u4e49\u8d44\u6e90\u3002</p>"},{"location":"start/underlay/#providernetwork","title":"\u521b\u5efa ProviderNetwork","text":"<p>ProviderNetwork \u63d0\u4f9b\u4e86\u4e3b\u673a\u7f51\u5361\u5230\u7269\u7406\u7f51\u7edc\u6620\u5c04\u7684\u62bd\u8c61\uff0c\u5c06\u540c\u5c5e\u4e00\u4e2a\u7f51\u7edc\u7684\u7f51\u5361\u8fdb\u884c\u7edf\u4e00\u7ba1\u7406\uff0c \u5e76\u89e3\u51b3\u5728\u590d\u6742\u73af\u5883\u4e0b\u540c\u673a\u5668\u591a\u7f51\u5361\u3001\u7f51\u5361\u540d\u4e0d\u4e00\u81f4\u3001\u5bf9\u5e94 Underlay \u7f51\u7edc\u4e0d\u4e00\u81f4\u7b49\u60c5\u51b5\u4e0b\u7684\u914d\u7f6e\u95ee\u9898\u3002</p> <p>\u521b\u5efa\u5982\u4e0b ProviderNetwork \u5e76\u5e94\u7528:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: ProviderNetwork\nmetadata:\n  name: net1\nspec:\n  defaultInterface: eth1\n  customInterfaces:\n    - interface: eth2\n      nodes:\n        - node1\n  excludeNodes:\n    - node2\n</code></pre> <p>\u6ce8\u610f\uff1aProviderNetwork \u8d44\u6e90\u540d\u79f0\u7684\u957f\u5ea6\u4e0d\u5f97\u8d85\u8fc7 12\u3002</p> <ul> <li><code>defaultInterface</code>: \u4e3a\u9ed8\u8ba4\u4f7f\u7528\u7684\u8282\u70b9\u7f51\u5361\u540d\u79f0\u3002 ProviderNetwork \u521b\u5efa\u6210\u529f\u540e\uff0c\u5404\u8282\u70b9\uff08\u9664 excludeNodes \u5916\uff09\u4e2d\u4f1a\u521b\u5efa\u540d\u4e3a br-net1\uff08\u683c\u5f0f\u4e3a <code>br-NAME</code>\uff09\u7684 OVS \u7f51\u6865\uff0c\u5e76\u5c06\u6307\u5b9a\u7684\u8282\u70b9\u7f51\u5361\u6865\u63a5\u81f3\u6b64\u7f51\u6865\u3002</li> <li><code>customInterfaces</code>: \u4e3a\u53ef\u9009\u9879\uff0c\u53ef\u9488\u5bf9\u7279\u5b9a\u8282\u70b9\u6307\u5b9a\u9700\u8981\u4f7f\u7528\u7684\u7f51\u5361\u3002</li> <li><code>excludeNodes</code>: \u53ef\u9009\u9879\uff0c\u7528\u4e8e\u6307\u5b9a\u4e0d\u6865\u63a5\u7f51\u5361\u7684\u8282\u70b9\u3002\u8be5\u5217\u8868\u4e2d\u7684\u8282\u70b9\u4f1a\u88ab\u6dfb\u52a0 <code>net1.provider-network.ovn.kubernetes.io/exclude=true</code> \u6807\u7b7e\u3002</li> </ul> <p>\u5176\u5b83\u8282\u70b9\u4f1a\u88ab\u6dfb\u52a0\u5982\u4e0b\u6807\u7b7e\uff1a</p> Key Value \u63cf\u8ff0 net1.provider-network.ovn.kubernetes.io/ready true \u8282\u70b9\u4e2d\u7684\u6865\u63a5\u5de5\u4f5c\u5df2\u5b8c\u6210\uff0cProviderNetwork \u5728\u8282\u70b9\u4e2d\u53ef\u7528 net1.provider-network.ovn.kubernetes.io/interface eth1 \u8282\u70b9\u4e2d\u88ab\u6865\u63a5\u7684\u7f51\u5361\u7684\u540d\u79f0 net1.provider-network.ovn.kubernetes.io/mtu 1500 \u8282\u70b9\u4e2d\u88ab\u6865\u63a5\u7684\u7f51\u5361\u7684 MTU <p>\u5982\u679c\u8282\u70b9\u7f51\u5361\u4e0a\u5df2\u7ecf\u914d\u7f6e\u4e86 IP\uff0c\u5219 IP \u5730\u5740\u548c\u7f51\u5361\u4e0a\u7684\u8def\u7531\u4f1a\u88ab\u8f6c\u79fb\u81f3\u5bf9\u5e94\u7684 OVS \u7f51\u6865\u3002</p>"},{"location":"start/underlay/#vlan","title":"\u521b\u5efa VLAN","text":"<p>Vlan \u63d0\u4f9b\u4e86\u5c06 Vlan Tag \u548c ProviderNetwork \u8fdb\u884c\u7ed1\u5b9a\u7684\u80fd\u529b\u3002</p> <p>\u521b\u5efa\u5982\u4e0b VLAN \u5e76\u5e94\u7528\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Vlan\nmetadata:\n  name: vlan1\nspec:\n  id: 0\n  provider: net1\n</code></pre> <ul> <li><code>id</code>: \u4e3a VLAN ID/Tag\uff0cKube-OVN \u4f1a\u5bf9\u5bf9\u8be5 Vlan \u4e0b\u7684\u6d41\u91cf\u589e\u52a0 Vlan \u6807\u7b7e\uff0c\u4e3a 0 \u65f6\u4e0d\u589e\u52a0\u4efb\u4f55\u6807\u7b7e\u3002</li> <li><code>provider</code>: \u4e3a\u9700\u8981\u4f7f\u7528\u7684 ProviderNetwork \u8d44\u6e90\u7684\u540d\u79f0\u3002\u591a\u4e2a VLAN \u53ef\u4ee5\u5f15\u7528\u540c\u4e00\u4e2a ProviderNetwork\u3002</li> </ul>"},{"location":"start/underlay/#subnet","title":"\u521b\u5efa Subnet","text":"<p>\u5c06 Vlan \u548c\u4e00\u4e2a\u5b50\u7f51\u7ed1\u5b9a\uff0c\u5982\u4e0b\u6240\u793a\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: subnet1\nspec:\nprotocol: IPv4\ncidrBlock: 172.17.0.0/16\ngateway: 172.17.0.1\nvlan: vlan1\n</code></pre> <p>\u5c06 <code>vlan</code> \u7684\u503c\u6307\u5b9a\u4e3a\u9700\u8981\u4f7f\u7528\u7684 VLAN \u540d\u79f0\u5373\u53ef\u3002\u591a\u4e2a Subnet \u53ef\u4ee5\u5f15\u7528\u540c\u4e00\u4e2a VLAN\u3002</p>"},{"location":"start/underlay/#_7","title":"\u5bb9\u5668\u521b\u5efa","text":"<p>\u53ef\u6309\u6b63\u5e38\u5bb9\u5668\u521b\u5efa\u65b9\u5f0f\u8fdb\u884c\u521b\u5efa\uff0c\u67e5\u770b\u5bb9\u5668 IP \u662f\u5426\u5728\u89c4\u5b9a\u8303\u56f4\u5185\uff0c\u4ee5\u53ca\u5bb9\u5668\u662f\u5426\u53ef\u4ee5\u548c\u7269\u7406\u7f51\u7edc\u4e92\u901a\u3002</p> <p>\u5982\u6709\u56fa\u5b9a IP \u9700\u6c42\uff0c\u53ef\u53c2\u8003 Pod \u56fa\u5b9a IP \u548c Mac</p>"},{"location":"start/underlay/#_8","title":"\u4f7f\u7528\u903b\u8f91\u7f51\u5173","text":"<p>\u5bf9\u4e8e\u7269\u7406\u7f51\u7edc\u4e0d\u5b58\u5728\u7f51\u5173\u7684\u60c5\u51b5\uff0cKube-OVN \u652f\u6301\u5728 Underlay \u6a21\u5f0f\u7684\u5b50\u7f51\u4e2d\u914d\u7f6e\u4f7f\u7528\u903b\u8f91\u7f51\u5173\u3002 \u82e5\u8981\u4f7f\u7528\u6b64\u529f\u80fd\uff0c\u8bbe\u7f6e\u5b50\u7f51\u7684 <code>spec.logicalGateway</code> \u4e3a <code>true</code> \u5373\u53ef\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n  name: subnet1\nspec:\n  protocol: IPv4\n  cidrBlock: 172.17.0.0/16\n  gateway: 172.17.0.1\n  vlan: vlan1\n  logicalGateway: true\n</code></pre> <p>\u5f00\u542f\u6b64\u529f\u80fd\u540e\uff0cPod \u4e0d\u4f7f\u7528\u5916\u90e8\u7f51\u5173\uff0c\u800c\u662f\u4f7f\u7528 Kube-OVN \u521b\u5efa\u7684\u903b\u8f91\u8def\u7531\u5668\uff08Logical Router\uff09\u5bf9\u4e8e\u8de8\u7f51\u6bb5\u901a\u4fe1\u8fdb\u884c\u8f6c\u53d1\u3002</p>"},{"location":"start/underlay/#underlay-overlay","title":"Underlay \u548c Overlay \u7f51\u7edc\u4e92\u901a","text":"<p>\u5982\u679c\u4e00\u4e2a\u96c6\u7fa4\u540c\u65f6\u5b58\u5728 Underlay \u548c Overlay \u5b50\u7f51\uff0c\u9ed8\u8ba4\u60c5\u51b5\u4e0b Overlay \u5b50\u7f51\u4e0b\u7684 Pod \u53ef\u4ee5\u901a\u8fc7\u7f51\u5173\u4ee5 NAT \u7684\u65b9\u5f0f\u8bbf\u95ee Underlay \u5b50\u7f51\u4e0b\u7684 Pod IP\u3002 \u5728 Underlay \u5b50\u7f51\u7684 Pod \u770b\u6765 Overlay \u5b50\u7f51\u7684\u5730\u5740\u662f\u4e00\u4e2a\u5916\u90e8\u7684\u5730\u5740\uff0c\u9700\u8981\u901a\u8fc7\u5e95\u5c42\u7269\u7406\u8bbe\u5907\u53bb\u8f6c\u53d1\uff0c\u4f46\u5e95\u5c42\u7269\u7406\u8bbe\u5907\u5e76\u4e0d\u6e05\u695a Overlay \u5b50\u7f51\u7684\u5730\u5740\u65e0\u6cd5\u8fdb\u884c\u8f6c\u53d1\u3002 \u56e0\u6b64 Underlay \u5b50\u7f51\u4e0b\u7684 Pod \u65e0\u6cd5\u901a\u8fc7 Pod IP \u76f4\u63a5\u8bbf\u95ee Overlay \u5b50\u7f51\u7684 Pod\u3002</p> <p>\u5982\u679c\u9700\u8981 Underlay \u548c Overlay \u4e92\u901a\u9700\u8981\u5c06\u5b50\u7f51\u7684 <code>u2oInterconnection</code> \u8bbe\u7f6e\u4e3a <code>true</code>\uff0c\u5728\u8fd9\u4e2a\u60c5\u51b5\u4e0b Kube-OVN \u4f1a\u989d\u5916\u4f7f\u7528\u4e00\u4e2a Underlay IP \u5c06 Underlay \u5b50\u7f51 \u548c <code>ovn-cluster</code> \u903b\u8f91\u8def\u7531\u5668\u8fde\u63a5\uff0c\u5e76\u8bbe\u7f6e\u5bf9\u5e94\u7684\u8def\u7531\u89c4\u5219\u5b9e\u73b0\u4e92\u901a\u3002 \u548c\u903b\u8f91\u7f51\u5173\u4e0d\u540c\uff0c\u8be5\u65b9\u6848\u53ea\u4f1a\u8fde\u63a5 Kube-OVN \u5185\u90e8\u7684 Underlay \u548c Overlay \u5b50\u7f51\uff0c\u5176\u4ed6\u8bbf\u95ee\u5916\u7f51\u7684\u6d41\u91cf\u8fd8\u662f\u4f1a\u901a\u8fc7\u7269\u7406\u7f51\u5173\u8fdb\u884c\u8f6c\u53d1\u3002</p>"},{"location":"start/underlay/#ip","title":"\u6307\u5b9a\u903b\u8f91\u7f51\u5173 IP","text":"<p>\u5f00\u542f\u4e92\u901a\u529f\u80fd\u540e\uff0c\u4f1a\u968f\u673a\u4ece subnet \u5185\u7684\u53d6\u4e00\u4e2a IP \u4f5c\u4e3a\u903b\u8f91\u7f51\u5173\uff0c\u5982\u679c\u9700\u8981\u6307\u5b9a Underlay Subnet \u7684\u903b\u8f91\u7f51\u5173\u53ef\u4ee5\u6307\u5b9a\u5b57\u6bb5 <code>u2oInterconnectionIP</code>\u3002</p>"},{"location":"start/underlay/#underlay-subnet-vpc","title":"\u6307\u5b9a Underlay Subnet \u8fde\u63a5\u7684\u81ea\u5b9a\u4e49 VPC","text":"<p>\u9ed8\u8ba4\u60c5\u51b5\u4e0b Underlay Subnet \u4f1a\u548c\u9ed8\u8ba4 VPC \u4e0a\u7684 Overlay Subnet \u4e92\u901a\uff0c\u5982\u679c\u8981\u6307\u5b9a\u548c\u67d0\u4e2a VPC \u4e92\u901a\uff0c\u5728 <code>u2oInterconnection</code> \u8bbe\u7f6e\u4e3a <code>true</code> \u540e\uff0c\u6307\u5b9a <code>subnet.spec.vpc</code> \u5b57\u6bb5\u4e3a\u8be5 VPC \u540d\u5b57\u5373\u53ef\u3002</p>"},{"location":"start/underlay/#_9","title":"\u6ce8\u610f\u4e8b\u9879","text":"<p>\u5982\u679c\u60a8\u4f7f\u7528\u7684\u8282\u70b9\u7f51\u5361\u4e0a\u914d\u7f6e\u6709 IP \u5730\u5740\uff0c\u4e14\u64cd\u4f5c\u7cfb\u7edf\u901a\u8fc7 Netplan \u914d\u7f6e\u7f51\u7edc\uff08\u5982 Ubuntu\uff09\uff0c\u5efa\u8bae\u60a8\u5c06 Netplan \u7684 renderer \u8bbe\u7f6e\u4e3a NetworkManager\uff0c\u5e76\u4e3a\u8282\u70b9\u7f51\u5361\u914d\u7f6e\u9759\u6001 IP \u5730\u5740\uff08\u5173\u95ed DHCP\uff09\uff1a</p> <pre><code>network:\nrenderer: NetworkManager\nethernets:\neth0:\ndhcp4: no\naddresses:\n- 172.16.143.129/24\nversion: 2\n</code></pre> <p>\u5982\u679c\u60a8\u8981\u4fee\u6539\u7f51\u5361\u7684 IP \u6216\u8def\u7531\u914d\u7f6e\uff0c\u9700\u8981\u5728\u4fee\u6539 netplan \u914d\u7f6e\u540e\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff1a</p> <pre><code>netplan generate\n\nnmcli connection reload netplan-eth0\nnmcli device set eth0 managed yes\n</code></pre> <p>\u6267\u884c\u4ee5\u4e0a\u547d\u4ee4\u540e\uff0cKube-OVN \u4f1a\u5c06\u7f51\u5361\u4e0a\u7684 IP \u53ca\u8def\u7531\u91cd\u65b0\u8f6c\u79fb\u81f3 OVS \u7f51\u6865\u3002</p> <p>\u5982\u679c\u60a8\u4f7f\u7528\u7684\u64cd\u4f5c\u7cfb\u7edf\u901a\u8fc7 NetworkManager \u7ba1\u7406\u7f51\u7edc\uff08\u5982 CentOS\uff09\uff0c\u5728\u4fee\u6539\u7f51\u5361\u914d\u7f6e\u540e\u9700\u8981\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff1a</p> <pre><code>nmcli connection reload eth0\nnmcli device set eth0 managed yes\nnmcli -t -f GENERAL.STATE device show eth0 | grep -qw unmanaged || nmcli device reapply eth0\n</code></pre> <p>\u6ce8\u610f\uff1a\u8282\u70b9\u7f51\u5361\u914d\u7f6e\u7684\u52a8\u6001\u4fee\u6539\u4ec5\u652f\u6301 IP \u548c\u8def\u7531\uff0c\u4e0d\u652f\u6301 MAC \u5730\u5740\u7684\u4fee\u6539\u3002</p>"},{"location":"start/underlay/#_10","title":"\u5df2\u77e5\u95ee\u9898","text":""},{"location":"start/underlay/#hairpin-pod","title":"\u7269\u7406\u7f51\u7edc\u5f00\u542f hairpin \u65f6 Pod \u7f51\u7edc\u5f02\u5e38","text":"<p>\u5f53\u7269\u7406\u7f51\u7edc\u5f00\u542f hairpin \u6216\u7c7b\u4f3c\u884c\u4e3a\u65f6\uff0c\u53ef\u80fd\u51fa\u73b0\u521b\u5efa Pod \u65f6\u7f51\u5173\u68c0\u67e5\u5931\u8d25\u3001Pod \u7f51\u7edc\u901a\u4fe1\u5f02\u5e38\u7b49\u95ee\u9898\u3002\u8fd9\u662f\u56e0\u4e3a OVS \u7f51\u6865\u9ed8\u8ba4\u7684 MAC \u5b66\u4e60\u529f\u80fd\u4e0d\u652f\u6301\u8fd9\u79cd\u7f51\u7edc\u73af\u5883\u3002</p> <p>\u8981\u89e3\u51b3\u6b64\u95ee\u9898\uff0c\u9700\u8981\u5173\u95ed hairpin\uff08\u6216\u4fee\u6539\u7269\u7406\u7f51\u7edc\u7684\u76f8\u5173\u914d\u7f6e\uff09\uff0c\u6216\u66f4\u65b0 Kube-OVN \u7248\u672c\u3002</p>"},{"location":"start/underlay/#pod-pod","title":"Pod \u6570\u91cf\u8f83\u591a\u65f6\u65b0\u5efa Pod \u7f51\u5173\u68c0\u67e5\u5931\u8d25","text":"<p>\u82e5\u540c\u4e00\u4e2a\u8282\u70b9\u4e0a\u8fd0\u884c\u7684 Pod \u6570\u91cf\u8f83\u591a\uff08\u5927\u4e8e 300\uff09\uff0c\u53ef\u80fd\u4f1a\u51fa\u73b0 ARP \u5e7f\u64ad\u5305\u7684 OVS \u6d41\u8868 resubmit \u6b21\u6570\u8d85\u8fc7\u4e0a\u9650\u5bfc\u81f4\u4e22\u5305\u7684\u73b0\u8c61\uff1a</p> <pre><code>2022-11-13T08:43:46.782Z|00222|ofproto_dpif_upcall(handler5)|WARN|Flow: arp,in_port=331,vlan_tci=0x0000,dl_src=00:00:00:25:eb:39,dl_dst=ff:ff:ff:ff:ff:ff,arp_spa=10.213.131.240,arp_tpa=10.213.159.254,arp_op=1,arp_sha=00:00:00:25:eb:39,arp_tha=ff:ff:ff:ff:ff:ff\n\nbridge(\"br-int\")\n----------------\n 0. No match.\n     &gt;&gt;&gt;&gt; received packet on unknown port 331 &lt;&lt;&lt;&lt;\n    drop\n\nFinal flow: unchanged\nMegaflow: recirc_id=0,eth,arp,in_port=331,dl_src=00:00:00:25:eb:39\nDatapath actions: drop\n2022-11-13T08:44:34.077Z|00224|ofproto_dpif_xlate(handler5)|WARN|over 4096 resubmit actions on bridge br-int while processing arp,in_port=13483,vlan_tci=0x0000,dl_src=00:00:00:59:ef:13,dl_dst=ff:ff:ff:ff:ff:ff,arp_spa=10.213.152.3,arp_tpa=10.213.159.254,arp_op=1,arp_sha=00:00:00:59:ef:13,arp_tha=ff:ff:ff:ff:ff:ff\n</code></pre> <p>\u8981\u89e3\u51b3\u6b64\u95ee\u9898\uff0c\u53ef\u4fee\u6539 OVN NB \u9009\u9879 <code>bcast_arp_req_flood</code> \u4e3a <code>false</code>\uff1a</p> <pre><code>kubectl ko nbctl set NB_Global . options:bcast_arp_req_flood=false\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"start/uninstall/","title":"\u5378\u8f7d","text":"<p>\u5982\u679c\u9700\u8981\u5220\u9664 Kube-OVN \u5e76\u66f4\u6362\u5176\u4ed6\u7f51\u7edc\u63d2\u4ef6\uff0c\u8bf7\u6309\u7167\u4e0b\u5217\u7684\u6b65\u9aa4\u5220\u9664\u5bf9\u5e94\u7684 Kube-OVN \u7ec4\u4ef6\u4ee5\u53ca OVS \u914d\u7f6e\uff0c\u4ee5\u907f\u514d\u5bf9\u5176\u4ed6\u7f51\u7edc\u63d2\u4ef6\u4ea7\u751f\u5e72\u6270\u3002 \u4e5f\u6b22\u8fce\u63d0 issue \u8054\u7cfb\u6211\u4eec\u53cd\u9988\u4e0d\u4f7f\u7528 Kube-OVN \u7684\u539f\u56e0\u5e2e\u52a9\u6211\u4eec\u6539\u8fdb\u3002</p>"},{"location":"start/uninstall/#kubernetes","title":"\u5220\u9664\u5728 Kubernetes \u4e2d\u521b\u5efa\u7684\u8d44\u6e90","text":"<p>\u4e0b\u8f7d\u4e0b\u9762\u7684\u811a\u672c\uff0c\u6267\u884c\u811a\u672c\u5220\u9664\u5728 Kubernetes \u4e2d\u521b\u5efa\u7684\u8d44\u6e90\uff1a</p> <pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/images/cleanup.sh\nbash cleanup.sh\n</code></pre>"},{"location":"start/uninstall/#_2","title":"\u6e05\u7406\u4e3b\u673a\u4e0a\u7684\u65e5\u5fd7\u548c\u914d\u7f6e\u6587\u4ef6","text":"<p>\u5728\u6bcf\u53f0\u673a\u5668\u4e0a\u6267\u884c\u4e0b\u5217\u64cd\u4f5c\uff0c\u6e05\u7406 ovsdb \u4ee5\u53ca openvswitch \u4fdd\u5b58\u7684\u914d\u7f6e\uff1a</p> <pre><code>rm -rf /var/run/openvswitch\nrm -rf /var/run/ovn\nrm -rf /etc/origin/openvswitch/\nrm -rf /etc/origin/ovn/\nrm -rf /etc/cni/net.d/00-kube-ovn.conflist\nrm -rf /etc/cni/net.d/01-kube-ovn.conflist\nrm -rf /var/log/openvswitch\nrm -rf /var/log/ovn\nrm -fr /var/log/kube-ovn\n</code></pre>"},{"location":"start/uninstall/#_3","title":"\u91cd\u542f\u8282\u70b9","text":"<p>\u91cd\u542f\u673a\u5668\u786e\u4fdd\u5bf9\u5e94\u7684\u7f51\u5361\u4fe1\u606f\uff0ciptable/ipset \u89c4\u5219\u5f97\u4ee5\u6e05\u9664\uff0c\u907f\u514d\u5bf9\u5176\u4ed6\u7f51\u7edc\u63d2\u4ef6\u7684\u5f71\u54cd\uff1a</p> <pre><code>reboot\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/","title":"Kube-OVN","text":"<p>Kube-OVN, a CNCF Sandbox Project, bridges the SDN into Cloud Native. It offers an advanced Container Network Fabric for Enterprises with the most functions, extreme performance and the easiest operation.</p> <p>Most Functions:</p> <p>If you miss the rich networking capabilities of the SDN age but are struggling to find them in the cloud-native age, Kube-OVN should be your best choice.</p> <p>Leveraging the proven capabilities of OVS/OVN in the SDN, Kube-OVN brings the rich capabilities of network virtualization to the cloud-native space. It currently supports Subnet Management, Static IP Allocation, Distributed/Centralized Gateways, Underlay/Overlay Hybrid Networks, VPC Multi-Tenant Networks, Cross-Cluster Interconnect, QoS Management, Multi-NIC Management, ACL, Traffic Mirroring, ARM Support, Windows Support, and many more.</p> <p>Extreme Performance:</p> <p>If you're concerned about the additional performance loss associated with container networks, then take a look at How Kube-OVN is doing everything it can to optimize performance.</p> <p>In the data plane, through a series of carefully optimized flow and kernel optimizations, and with emerging technologies such as eBPF, DPDK and SmartNIC Offload, Kube-OVN can approximate or exceed host network performance in terms of latency and throughput.</p> <p>In the control plane, Kube-OVN can support large-scale clusters of thousands of nodes and tens of thousands of Pods through the tailoring of OVN upstream flow tables and the use and tuning of various caching techniques.</p> <p>In addition, Kube-OVN is continuously optimizing the usage of resources such as CPU and memory to accommodate resource-limited scenarios such as the edge.</p> <p>Easiest Operation:</p> <p>If you're worried about container network operations, Kube-OVN has a number of built-in tools to help you simplify your operations.</p> <p>Kube-OVN provides one-click installation scripts to help users quickly build production-ready container networks. Also built-in rich monitoring metrics and Grafana dashboard help users to quickly set up monitoring system.</p> <p>Powerful command line tools simplify daily operations and maintenance for users. By combining with Cilium, users can enhance the observability of their networks with eBPF capabilities. In addition, the ability to mirror traffic makes it easy to customize traffic monitoring and interface with traditional NPM systems.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/contact/","title":"Contact US","text":"<p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/accelerate-intra-node-tcp-with-ebpf/","title":"Accelerate TCP Communication in Node with eBPF","text":"<p>At some edge and 5G scenarios, there will be a lot of TCP communication between Pods on the same node. By using the open source istio-tcpip-bypass project from Intel, Pods can use the ability of eBPF to bypass the host's TCP/IP protocol stack and communicate directly through sockets, thereby greatly reducing latency and improving throughput.</p>"},{"location":"en/advance/accelerate-intra-node-tcp-with-ebpf/#basic-principle","title":"Basic Principle","text":"<p>At present, two Pods on the same host need to go through a lot of network stacks, including TCP/IP, netfilter, OVS, etc., as shown in the following figure:</p> <p></p> <p>istio-tcpip-bypass plugin can automatically analyze and identify TCP communication within the same host, and bypass the complex kernel stack so that socket data transmission can be performed directly to reduce network stack processing overhead, as shown in the following figure:</p> <p></p> <p>Due to the fact that this component can automatically identify TCP communication within the same host and optimize it. In the Service Mesh environment based on the proxy mode, this component can also enhance the performance of Service Mesh.</p> <p>For more technical implementation details, please refer to Tanzu Service Mesh Acceleration using eBPF.</p>"},{"location":"en/advance/accelerate-intra-node-tcp-with-ebpf/#prerequisites","title":"Prerequisites","text":"<p>eBPF requires a kernel version of at least 5.4.0-74-generic. It is recommended to use Ubuntu 20.04 and Linux 5.4.0-74-generic kernel version for testing.</p>"},{"location":"en/advance/accelerate-intra-node-tcp-with-ebpf/#experimental-steps","title":"Experimental Steps","text":"<p>Deploy two performance test Pods on the same node. If there are multiple machines in the cluster, you need to specify <code>nodeSelector</code>:</p> <pre><code># kubectl create deployment perf --image=kubeovn/perf:dev --replicas=2\ndeployment.apps/perf created\n# kubectl get pod -o wide\nNAME                    READY   STATUS    RESTARTS   AGE   IP           NODE     NOMINATED NODE   READINESS GATES\nperf-7697bc6ddf-b2cpv   1/1     Running   0          28s   100.64.0.3   sealos   &lt;none&gt;           &lt;none&gt;\nperf-7697bc6ddf-p2xpt   1/1     Running   0          28s   100.64.0.2   sealos   &lt;none&gt;           &lt;none&gt;\n</code></pre> <p>Enter one of the Pods to start the qperf server, and start the qperf client in another Pod for performance testing:</p> <pre><code># kubectl exec -it perf-7697bc6ddf-b2cpv sh\n/ # qperf\n\n# kubectl exec -it perf-7697bc6ddf-p2xpt sh\n/ # qperf -t 60 100.64.0.3 -ub -oo msg_size:1:16K:*4 -vu tcp_lat tcp_bw\n</code></pre> <p>Deploy the istio-tcpip-bypass plugin:</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/intel/istio-tcpip-bypass/main/bypass-tcpip-daemonset.yaml\n</code></pre> <p>Enter the perf client container again for performance testing:</p> <pre><code># kubectl exec -it perf-7697bc6ddf-p2xpt sh\n/ # qperf -t 60 100.64.0.3 -ub -oo msg_size:1:16K:*4 -vu tcp_lat tcp_bw\n</code></pre>"},{"location":"en/advance/accelerate-intra-node-tcp-with-ebpf/#test-results","title":"Test Results","text":"<p>According to the test results, the TCP latency will decrease by 40% ~ 60% under different packet sizes, and the throughput will increase by 40% ~ 80% when the packet size is greater than 1024 bytes.</p> Packet Size (byte) eBPF tcp_lat (us) Default tcp_lat (us) eBPF tcp_bw (Mb/s) Default tcp_bw(Mb/s) 1 20.2 44.5 1.36 4.27 4 20.2 48.7 5.48 16.7 16 19.6 41.6 21.7 63.5 64 18.8 41.3 96.8 201 256 19.2 36 395 539 1024 18.3 42.4 1360 846 4096 16.5 62.6 4460 2430 16384 20.2 58.8 9600 6900 <p>In the hardware environment under test, when the packet size is less than 512 bytes, the throughput indicator optimized by eBPF is lower than the throughput under the default configuration. This situation may be related to the TCP aggregation optimization of the network card under the default configuration. If the application scenario is sensitive to small packet throughput, you need to test in the corresponding environment Determine whether to enable eBPF optimization. We will also optimize the throughput of eBPF TCP small packet scenarios in the future.</p>"},{"location":"en/advance/accelerate-intra-node-tcp-with-ebpf/#references","title":"References","text":"<ol> <li>istio-tcpip-bypass</li> <li>Deep Dive TCP/IP Bypass with eBPF in Service Mesh</li> <li>Tanzu Service Mesh Acceleration using eBPF</li> </ol> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/cilium-hubble-observe/","title":"Cilium Network Traffic Observation","text":"<p>Kube-OVN supports Cilium integration, please refer to Cilium integration for details.</p> <p>Cilium provides rich network traffic observation capabilities, and the flow observability is provided by Hubble. Hubble can observe the traffic across nodes, clusters, and even multi-cluster scenarios.</p>"},{"location":"en/advance/cilium-hubble-observe/#install-hubble","title":"Install Hubble","text":"<p>In the default Cilium integration installation, the Hubble related components are not installed, so to support traffic observation, you need to supplement the installation of Hubble on the environment.</p> <p>Execute the following command to install Hubble using helm:</p> <pre><code>helm upgrade cilium cilium/cilium --version 1.11.6 \\\n--namespace kube-system \\\n--reuse-values \\\n--set hubble.relay.enabled=true \\\n--set hubble.ui.enabled=true\n</code></pre> <p>After installing Hubble, execute <code>cilium status</code> to check the status of the component and confirm that the installation is successful.</p> <pre><code># cilium status\n/\u00af\u00af\\\n/\u00af\u00af\\__/\u00af\u00af\\    Cilium:         OK\n \\__/\u00af\u00af\\__/    Operator:       OK\n /\u00af\u00af\\__/\u00af\u00af\\    Hubble:         OK\n \\__/\u00af\u00af\\__/    ClusterMesh:    disabled\n    \\__/\n\nDeployment        hubble-relay       Desired: 1, Ready: 1/1, Available: 1/1\nDeployment        cilium-operator    Desired: 2, Ready: 2/2, Available: 2/2\nDaemonSet         cilium             Desired: 2, Ready: 2/2, Available: 2/2\nDeployment        hubble-ui          Desired: 1, Ready: 1/1, Available: 1/1\nContainers:       cilium             Running: 2\nhubble-ui          Running: 1\nhubble-relay       Running: 1\ncilium-operator    Running: 2\nCluster Pods:     16/17 managed by Cilium\nImage versions    hubble-relay       quay.io/cilium/hubble-relay:v1.11.6@sha256:fd9034a2d04d5b973f1e8ed44f230ea195b89c37955ff32e34e5aa68f3ed675a: 1\ncilium-operator    quay.io/cilium/operator-generic:v1.11.6@sha256:9f6063c7bcaede801a39315ec7c166309f6a6783e98665f6693939cf1701bc17: 2\ncilium             quay.io/cilium/cilium:v1.11.6@sha256:f7f93c26739b6641a3fa3d76b1e1605b15989f25d06625260099e01c8243f54c: 2\nhubble-ui          quay.io/cilium/hubble-ui:v0.9.0@sha256:0ef04e9a29212925da6bdfd0ba5b581765e41a01f1cc30563cef9b30b457fea0: 1\nhubble-ui          quay.io/cilium/hubble-ui-backend:v0.9.0@sha256:000df6b76719f607a9edefb9af94dfd1811a6f1b6a8a9c537cba90bf12df474b: 1\napple@bogon cilium %\n</code></pre> <p>After installing the Hubble component, you need to install the command line to view the traffic information in the environment. Execute the following command to install Hubble CLI:</p> <pre><code>curl -L --fail --remote-name-all https://github.com/cilium/hubble/releases/download/v0.10.0/hubble-linux-amd64.tar.gz\nsudo tar xzvfC hubble-linux-amd64.tar.gz /usr/local/bin\n</code></pre>"},{"location":"en/advance/cilium-hubble-observe/#deploy-and-test","title":"Deploy and test","text":"<p>Cilium offers a traffic test deployment solution, you can directly use the official deployment solution to deploy the test.</p> <p>Execute the command <code>cilium connectivity test</code>, Cilium will automatically create the <code>cilium-test</code> namespace, and deploy the test under cilium-test.</p> <p>After the normal deployment, you can view the resource information under the <code>cilium-test</code> namespace, as follows:</p> <pre><code># kubectl get all -n cilium-test\nNAME                                  READY   STATUS    RESTARTS   AGE\npod/client-7df6cfbf7b-z5t2j           1/1     Running   0          21s\npod/client2-547996d7d8-nvgxg          1/1     Running   0          21s\npod/echo-other-node-d79544ccf-hl4gg   2/2     Running   0          21s\npod/echo-same-node-5d466d5444-ml7tc   2/2     Running   0          21s\n\nNAME                      TYPE       CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE\nservice/echo-other-node   NodePort   10.109.58.126   &lt;none&gt;        8080:32269/TCP   21s\nservice/echo-same-node    NodePort   10.108.70.32    &lt;none&gt;        8080:32490/TCP   21s\n\nNAME                              READY   UP-TO-DATE   AVAILABLE   AGE\ndeployment.apps/client            1/1     1            1           21s\ndeployment.apps/client2           1/1     1            1           21s\ndeployment.apps/echo-other-node   1/1     1            1           21s\ndeployment.apps/echo-same-node    1/1     1            1           21s\n\nNAME                                        DESIRED   CURRENT   READY   AGE\nreplicaset.apps/client-7df6cfbf7b           1         1         1       21s\nreplicaset.apps/client2-547996d7d8          1         1         1       21s\nreplicaset.apps/echo-other-node-d79544ccf   1         1         1       21s\nreplicaset.apps/echo-same-node-5d466d5444   1         1         1       21s\n</code></pre>"},{"location":"en/advance/cilium-hubble-observe/#use-the-command-line-to-observe-traffic","title":"Use the command line to observe traffic","text":"<p>By default, the network traffic observation only provides the traffic observed by the Cilium agent on each node.</p> <p>Execute the <code>hubble observe</code> command in the Cilium agent pod under the <code>kube-system namespace</code> to view the traffic information on the node.</p> <pre><code># kubectl get pod -n kube-system -o wide\nNAME                                             READY   STATUS    RESTARTS   AGE     IP           NODE                     NOMINATED NODE   READINESS GATES\ncilium-d6h56                                     1/1     Running   0          2d20h   172.18.0.2   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;\ncilium-operator-5887f78bbb-c7sb2                 1/1     Running   0          2d20h   172.18.0.2   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;\ncilium-operator-5887f78bbb-wj8gt                 1/1     Running   0          2d20h   172.18.0.3   kube-ovn-control-plane   &lt;none&gt;           &lt;none&gt;\ncilium-tq5xb                                     1/1     Running   0          2d20h   172.18.0.3   kube-ovn-control-plane   &lt;none&gt;           &lt;none&gt;\nkube-ovn-pinger-7lgk8                            1/1     Running   0          21h     10.16.0.19   kube-ovn-control-plane   &lt;none&gt;           &lt;none&gt;\nkube-ovn-pinger-msvcn                            1/1     Running   0          21h     10.16.0.18   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;\n\n# kubectl exec -it -n kube-system cilium-d6h56 -- bash\nroot@kube-ovn-worker:/home/cilium# hubble observe --from-namespace kube-system\nJul 29 03:24:25.551: kube-system/kube-ovn-pinger-msvcn:35576 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: ACK, PSH)\nJul 29 03:24:25.561: kube-system/kube-ovn-pinger-msvcn:35576 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: RST)\nJul 29 03:24:25.561: kube-system/kube-ovn-pinger-msvcn:35576 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: ACK, RST)\nJul 29 03:24:25.572: kube-system/kube-ovn-pinger-msvcn:35578 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: SYN)\nJul 29 03:24:25.572: kube-system/kube-ovn-pinger-msvcn:35578 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: ACK)\nJul 29 03:24:25.651: kube-system/kube-ovn-pinger-msvcn:35578 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: ACK, PSH)\nJul 29 03:24:25.661: kube-system/kube-ovn-pinger-msvcn:35578 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: RST)\nJul 29 03:24:25.661: kube-system/kube-ovn-pinger-msvcn:35578 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: ACK, RST)\nJul 29 03:24:25.761: kube-system/kube-ovn-pinger-msvcn:52004 -&gt; 172.18.0.3:6443 to-stack FORWARDED (TCP Flags: ACK, PSH)\nJul 29 03:24:25.779: kube-system/kube-ovn-pinger-msvcn -&gt; kube-system/kube-ovn-pinger-7lgk8 to-stack FORWARDED (ICMPv4 EchoRequest)\nJul 29 03:24:25.779: kube-system/kube-ovn-pinger-msvcn &lt;- kube-system/kube-ovn-pinger-7lgk8 to-endpoint FORWARDED (ICMPv4 EchoReply)\nJul 29 03:24:25.866: kube-system/hubble-ui-7596f7ff6f-7j6f2:55836 &lt;- kube-system/hubble-relay-959988db5-zc5vv:4245 to-stack FORWARDED (TCP Flags: ACK)\nJul 29 03:24:25.866: kube-system/hubble-ui-7596f7ff6f-7j6f2:55836 &lt;- kube-system/hubble-relay-959988db5-zc5vv:80 to-endpoint FORWARDED (TCP Flags: ACK)\nJul 29 03:24:25.866: kube-system/hubble-ui-7596f7ff6f-7j6f2:55836 -&gt; kube-system/hubble-relay-959988db5-zc5vv:4245 to-stack FORWARDED (TCP Flags: ACK)\nJul 29 03:24:25.866: kube-system/hubble-ui-7596f7ff6f-7j6f2:55836 -&gt; kube-system/hubble-relay-959988db5-zc5vv:4245 to-endpoint FORWARDED (TCP Flags: ACK)\nJul 29 03:24:25.975: kube-system/kube-ovn-pinger-7lgk8 -&gt; kube-system/kube-ovn-pinger-msvcn to-endpoint FORWARDED (ICMPv4 EchoRequest)\nJul 29 03:24:25.975: kube-system/kube-ovn-pinger-7lgk8 &lt;- kube-system/kube-ovn-pinger-msvcn to-stack FORWARDED (ICMPv4 EchoReply)\nJul 29 03:24:25.979: kube-system/kube-ovn-pinger-msvcn -&gt; 172.18.0.3 to-stack FORWARDED (ICMPv4 EchoRequest)\nJul 29 03:24:26.037: kube-system/coredns-6d4b75cb6d-lbgjg:36430 -&gt; 172.18.0.3:6443 to-stack FORWARDED (TCP Flags: ACK)\nJul 29 03:24:26.282: kube-system/kube-ovn-pinger-msvcn -&gt; 172.18.0.2 to-stack FORWARDED (ICMPv4 EchoRequest)\n</code></pre> <p>After deploying Hubble Relay, Hubble can provide complete cluster-wide network traffic observation.</p>"},{"location":"en/advance/cilium-hubble-observe/#configure-port-forwarding","title":"Configure port forwarding","text":"<p>In order to access the Hubble API normally, you need to create a port forwarding to forward the local request to the Hubble Service. You can execute the <code>kubectl port-forward deployment/hubble-relay -n kube-system 4245:4245</code> command to open the port forwarding in the current terminal.</p> <p>The port forwarding configuration can refer to Port Forwarding.</p> <p><code>kubectl port-forward</code> is a blocking command, you can open a new terminal to execute the following command to observe the traffic information.</p> <p>After configuring the port forwarding, execute the <code>hubble status</code> command in the terminal. If there is an output similar to the following, the port forwarding configuration is correct, and you can use the command line to observe the traffic.</p> <pre><code># hubble status\nHealthcheck (via localhost:4245): Ok\nCurrent/Max Flows: 8,190/8,190 (100.00%)\nFlows/s: 22.86\nConnected Nodes: 2/2\n</code></pre>"},{"location":"en/advance/cilium-hubble-observe/#use-the-command-line-to-observe-traffic_1","title":"Use the command line to observe traffic","text":"<p>Execute the <code>hubble observe</code> command in the terminal to view the traffic information of the cluster.</p> <p>The traffic observed by the <code>cilium-test</code> namespace is as follows:</p> <p></p> <p>Pay attention to the <code>hubble observe</code> command display result, which is the traffic information queried when the current command line is executed. Executing the command line multiple times can view different traffic information. For more detailed observation information, you can execute the <code>hubble help observe</code> command to view the detailed usage of Hubble CLI.</p>"},{"location":"en/advance/cilium-hubble-observe/#use-ui-to-observe-traffic","title":"Use UI to observe traffic","text":"<p>Execute the <code>cilium status</code> command to confirm that the Hubble UI has been successfully installed. In the second step of the Hubble installation, the installation of the UI has been supplemented.</p> <p>Execute the command <code>cilium hubble ui</code> to automatically create port forwarding and map the <code>hubble-ui service</code> to the local port.</p> <p>When the command is executed normally, the local browser will be automatically opened and jump to the Hubble UI interface. If it does not jump automatically, enter <code>http://localhost:12000</code> in the browser to open the UI observation interface.</p> <p>On the top left of the UI, select the <code>cilium-test</code> namespace to view the test traffic information provided by Cilium. </p>"},{"location":"en/advance/cilium-hubble-observe/#hubble-traffic-monitoring","title":"Hubble Traffic Monitoring","text":"<p>Hubble component provides monitoring of Pod network behavior in the cluster. In order to support viewing the monitoring data provided by Hubble, you need to enable monitoring statistics.</p> <p>Refer to the following command to supplement the <code>hubble.metrics.enabled</code> configuration item:</p> <pre><code>helm upgrade cilium cilium/cilium --version 1.11.6 \\\n--namespace kube-system \\\n--reuse-values \\\n--set hubble.relay.enabled=true \\\n--set hubble.ui.enabled=true \\\n--set hubble.metrics.enabled=\"{dns,drop,tcp,flow,icmp,http}\"\n</code></pre> <p>After the deployment is completed, you can view the monitoring data provided by Hubble through the <code>hubble-metrics</code> service. Execute the following command to view the monitoring data:</p> <pre><code># curl 172.18.0.2:9091/metrics\n# HELP hubble_drop_total Number of drops\n# TYPE hubble_drop_total counter\nhubble_drop_total{protocol=\"ICMPv6\",reason=\"Unsupported L3 protocol\"} 2\n# HELP hubble_flows_processed_total Total number of flows processed\n# TYPE hubble_flows_processed_total counter\nhubble_flows_processed_total{protocol=\"ICMPv4\",subtype=\"to-endpoint\",type=\"Trace\",verdict=\"FORWARDED\"} 335\nhubble_flows_processed_total{protocol=\"ICMPv4\",subtype=\"to-stack\",type=\"Trace\",verdict=\"FORWARDED\"} 335\nhubble_flows_processed_total{protocol=\"ICMPv6\",subtype=\"\",type=\"Drop\",verdict=\"DROPPED\"} 2\nhubble_flows_processed_total{protocol=\"TCP\",subtype=\"to-endpoint\",type=\"Trace\",verdict=\"FORWARDED\"} 8282\nhubble_flows_processed_total{protocol=\"TCP\",subtype=\"to-stack\",type=\"Trace\",verdict=\"FORWARDED\"} 6767\nhubble_flows_processed_total{protocol=\"UDP\",subtype=\"to-endpoint\",type=\"Trace\",verdict=\"FORWARDED\"} 1642\nhubble_flows_processed_total{protocol=\"UDP\",subtype=\"to-stack\",type=\"Trace\",verdict=\"FORWARDED\"} 1642\n# HELP hubble_icmp_total Number of ICMP messages\n# TYPE hubble_icmp_total counter\nhubble_icmp_total{family=\"IPv4\",type=\"EchoReply\"} 335\nhubble_icmp_total{family=\"IPv4\",type=\"EchoRequest\"} 335\nhubble_icmp_total{family=\"IPv4\",type=\"RouterSolicitation\"} 2\n# HELP hubble_tcp_flags_total TCP flag occurrences\n# TYPE hubble_tcp_flags_total counter\nhubble_tcp_flags_total{family=\"IPv4\",flag=\"FIN\"} 2043\nhubble_tcp_flags_total{family=\"IPv4\",flag=\"RST\"} 301\nhubble_tcp_flags_total{family=\"IPv4\",flag=\"SYN\"} 1169\nhubble_tcp_flags_total{family=\"IPv4\",flag=\"SYN-ACK\"} 1169\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/cilium-networkpolicy/","title":"Cilium NetworkPolicy Support","text":"<p>Kube-OVN currently supports integration with Cilium, and the specific operation can refer to Cilium integration.</p> <p>After integrating Cilium, you can use Cilium's excellent network policy capabilities to control the access of Pods in the cluster.The following documents provide integration verification of Cilium L3 and L4 network policy capabilities.</p>"},{"location":"en/advance/cilium-networkpolicy/#verification-steps","title":"Verification Steps","text":""},{"location":"en/advance/cilium-networkpolicy/#create-test-pod","title":"Create test Pod","text":"<p>Create namespace <code>test</code>. Refer to the following yaml, create Pod with label <code>app=test</code> in namespace <code>test</code> as the destination Pod for testing access.</p> <pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nlabels:\napp: test\nname: test\nnamespace: test\nspec:\nreplicas: 1\nselector:\nmatchLabels:\napp: test\nstrategy:\nrollingUpdate:\nmaxSurge: 25%\nmaxUnavailable: 25%\ntype: RollingUpdate\ntemplate:\nmetadata:\nlabels:\napp: test\nspec:\ncontainers:\n- image: docker.io/library/nginx:alpine\nimagePullPolicy: IfNotPresent\nname: nginx\n</code></pre> <p>Similarly, refer to the following yaml, create Pod with label <code>app=dynamic</code> in namespace <code>default</code> as the Pod for testing access.</p> <pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nlabels:\napp: dynamic\nname: dynamic\nnamespace: default\nspec:\nreplicas: 2\nselector:\nmatchLabels:\napp: dynamic\nstrategy:\nrollingUpdate:\nmaxSurge: 25%\nmaxUnavailable: 25%\ntype: RollingUpdate\ntemplate:\nmetadata:\ncreationTimestamp: null\nlabels:\napp: dynamic\nspec:\ncontainers:\n- image: docker.io/library/nginx:alpine\nimagePullPolicy: IfNotPresent\nname: nginx\n</code></pre> <p>View the test Pod and Label information:</p> <pre><code># kubectl get pod -o wide --show-labels\nNAME                         READY   STATUS    RESTARTS   AGE   IP           NODE                     NOMINATED NODE   READINESS GATES   LABELS\ndynamic-7d8d7874f5-9v5c4     1/1     Running   0          28h   10.16.0.35   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;            app=dynamic,pod-template-hash=7d8d7874f5\ndynamic-7d8d7874f5-s8z2n     1/1     Running   0          28h   10.16.0.36   kube-ovn-control-plane   &lt;none&gt;           &lt;none&gt;            app=dynamic,pod-template-hash=7d8d7874f5\n# kubectl get pod -o wide -n test --show-labels\nNAME                           READY   STATUS    RESTARTS   AGE     IP           NODE                     NOMINATED NODE   READINESS GATES   LABELS\ndynamic-7d8d7874f5-6dsg6       1/1     Running   0          7h20m   10.16.0.2    kube-ovn-control-plane   &lt;none&gt;           &lt;none&gt;            app=dynamic,pod-template-hash=7d8d7874f5\ndynamic-7d8d7874f5-tjgtp       1/1     Running   0          7h46m   10.16.0.42   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;            app=dynamic,pod-template-hash=7d8d7874f5\nlabel-test1-77b6764857-swq4k   1/1     Running   0          3h43m   10.16.0.12   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;            app=test1,pod-template-hash=77b6764857\n\n// As the destination Pod for testing access.\ntest-54c98bc466-mft5s          1/1     Running   0          8h      10.16.0.41   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;            app=test,pod-template-hash=54c98bc466\n</code></pre>"},{"location":"en/advance/cilium-networkpolicy/#l3-network-policy-test","title":"L3 Network Policy Test","text":"<p>Refer to the following yaml, create <code>CiliumNetworkPolicy</code> resource:</p> <pre><code>apiVersion: \"cilium.io/v2\"\nkind: CiliumNetworkPolicy\nmetadata:\nname: \"l3-rule\"\nnamespace: test\nspec:\nendpointSelector:\nmatchLabels:\napp: test\ningress:\n- fromEndpoints:\n- matchLabels:\napp: dynamic\n</code></pre> <p>At this point, the test Pod in the default namespace cannot access the destination Pod, but the test Pod to the destination Pod in the test namespace is accessible.</p> <p>Test results in the default namespace:</p> <pre><code># kubectl exec -it dynamic-7d8d7874f5-9v5c4 -- bash\nbash-5.0# ping -c 3 10.16.0.41\nPING 10.16.0.41 (10.16.0.41): 56 data bytes\n\n--- 10.16.0.41 ping statistics ---\n3 packets transmitted, 0 packets received, 100% packet loss\n</code></pre> <p>Test results in the test namespace:</p> <pre><code># kubectl exec -it -n test dynamic-7d8d7874f5-6dsg6 -- bash\nbash-5.0# ping -c 3 10.16.0.41\nPING 10.16.0.41 (10.16.0.41): 56 data bytes\n64 bytes from 10.16.0.41: seq=0 ttl=64 time=2.558 ms\n64 bytes from 10.16.0.41: seq=1 ttl=64 time=0.223 ms\n64 bytes from 10.16.0.41: seq=2 ttl=64 time=0.304 ms\n\n--- 10.16.0.41 ping statistics ---\n3 packets transmitted, 3 packets received, 0% packet loss\nround-trip min/avg/max = 0.223/1.028/2.558 ms\n</code></pre> <p>Look at the Cilium official document explanation, the <code>CiliumNetworkPolicy</code> resource limits the control at the <code>namespace</code> level. For more information, please refer to Cilium Limitations.</p> <p>If there is a network policy rule match, only the Pod in the same namespace can access according to the rule, and the Pod in the other namespace is denied access by default.</p> <p>If you want to implement cross-namespace access, you need to specify the namespace information in the rule.</p> <p>Refer to the document, modify the <code>CiliumNetworkPolicy</code> resource, and add namespace information:</p> <pre><code>  ingress:\n- fromEndpoints:\n- matchLabels:\napp: dynamic\nk8s:io.kubernetes.pod.namespace: default    // control the Pod access in other namespace\n</code></pre> <p>Look at the modified <code>CiliumNetworkPolicy</code> resource information:</p> <pre><code># kubectl get cnp -n test  -o yaml l3-rule\napiVersion: cilium.io/v2\nkind: CiliumNetworkPolicy\nmetadata:\n  name: l3-rule\n  namespace: test\nspec:\n  endpointSelector:\n    matchLabels:\n      app: test\ningress:\n  - fromEndpoints:\n    - matchLabels:\n        app: dynamic\n    - matchLabels:\n        app: dynamic\n        k8s:io.kubernetes.pod.namespace: default\n</code></pre> <p>Test the Pod access in the default namespace again, and the destination Pod access is normal:</p> <pre><code># kubectl exec -it dynamic-7d8d7874f5-9v5c4 -n test -- bash\nbash-5.0# ping -c 3 10.16.0.41\nPING 10.16.0.41 (10.16.0.41): 56 data bytes\n64 bytes from 10.16.0.41: seq=0 ttl=64 time=2.383 ms\n64 bytes from 10.16.0.41: seq=1 ttl=64 time=0.115 ms\n64 bytes from 10.16.0.41: seq=2 ttl=64 time=0.142 ms\n\n--- 10.16.0.41 ping statistics ---\n3 packets transmitted, 3 packets received, 0% packet loss\nround-trip min/avg/max = 0.115/0.880/2.383 ms\n</code></pre> <p>Using the standard Kubernetes network policy networkpolicy, the test results show that Cilium also restricts access within the same namespace, and cross-namespace access is prohibited.</p> <p>It is different from Kube-OVN implementation. Kube-OVN supports standard k8s network policy, which restricts the destination Pod in a specific namespace, but there is no namespace restriction on the source Pod. Any Pod that meets the restriction rules in any namespace can access the destination Pod.</p>"},{"location":"en/advance/cilium-networkpolicy/#l4-network-policy-test","title":"L4 Network Policy Test","text":"<p>Refer to the following yaml, create <code>CiliumNetworkPolicy</code> resource:</p> <pre><code>apiVersion: \"cilium.io/v2\"\nkind: CiliumNetworkPolicy\nmetadata:\nname: \"l4-rule\"\nnamespace: test\nspec:\nendpointSelector:\nmatchLabels:\napp: test\ningress:\n- fromEndpoints:\n- matchLabels:\napp: dynamic\ntoPorts:\n- ports:\n- port: \"80\"\nprotocol: TCP\n</code></pre> <p>Test the access of the Pod that meets the network policy rules in the same namespace</p> <pre><code># kubectl exec -it -n test dynamic-7d8d7874f5-6dsg6 -- bash\nbash-5.0# ping -c 3 10.16.0.41\nPING 10.16.0.41 (10.16.0.41): 56 data bytes\n\n--- 10.16.0.41 ping statistics ---\n3 packets transmitted, 0 packets received, 100% packet loss\nbash-5.0#\nbash-5.0# curl 10.16.0.41:80\n&lt;html&gt;\n&lt;head&gt;\n        &lt;title&gt;Hello World!&lt;/title&gt;\n        &lt;link href='//fonts.googleapis.com/css?family=Open+Sans:400,700' rel='stylesheet' type='text/css'&gt;\n        &lt;style&gt;\n        body {\nbackground-color: white;\ntext-align: center;\npadding: 50px;\nfont-family: \"Open Sans\",\"Helvetica Neue\",Helvetica,Arial,sans-serif;\n}\n#logo {\nmargin-bottom: 40px;\n}\n&lt;/style&gt;\n&lt;/head&gt;\n&lt;body&gt;\n                &lt;h1&gt;Hello World!&lt;/h1&gt;\n                                &lt;h3&gt;Links found&lt;/h3&gt;\n        &lt;h3&gt;I am on  test-54c98bc466-mft5s&lt;/h3&gt;\n        &lt;h3&gt;Cookie                  =&lt;/h3&gt;\n                                        &lt;b&gt;KUBERNETES&lt;/b&gt; listening in 443 available at tcp://10.96.0.1:443&lt;br /&gt;\n                                                &lt;h3&gt;my name is hanhouchao!&lt;/h3&gt;\n                        &lt;h3&gt; RequestURI='/'&lt;/h3&gt;\n&lt;/body&gt;\n&lt;/html&gt;\n</code></pre> <p>The Pod that does not meet the network policy rules in the same namespace cannot access</p> <pre><code># kubectl exec -it -n test label-test1-77b6764857-swq4k -- bash\nbash-5.0# ping -c 3 10.16.0.41\nPING 10.16.0.41 (10.16.0.41): 56 data bytes\n\n--- 10.16.0.41 ping statistics ---\n3 packets transmitted, 0 packets received, 100% packet loss\nbash-5.0#\nbash-5.0# curl -v 10.16.0.41:80 --connect-timeout 10\n*   Trying 10.16.0.41:80...\n* After 10000ms connect time, move on!\n* connect to 10.16.0.41 port 80 failed: Operation timed out\n* Connection timeout after 10001 ms\n* Closing connection 0\ncurl: (28) Connection timeout after 10001 ms\n</code></pre> <p>After the network policy takes effect, cross-namespace access is still prohibited, which is consistent with the L3 network policy test results.</p> <p>After the L4 network policy takes effect, ping cannot be used, but TCP access that meets the policy rules can be executed normally.</p> <p>About the restriction of ICMP, please refer to the official description L4 Limitation Description.</p>"},{"location":"en/advance/cilium-networkpolicy/#l7-network-policy-test","title":"L7 Network Policy Test","text":"<p>chaining mode, L7 network policy currently has problems. In the Cilium official document, there is an explanation for this situation, please refer to Generic Veth Chaining.</p> <p>This problem is tracked using issue 12454, and it has not been resolved yet.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/dhcp/","title":"DHCP","text":"<p>When using SR-IOV or DPDK type networks, KubeVirt's built-in DHCP does not work in this network mode. Kube-OVN can use the DHCP capabilities of OVN to set DHCP options at the subnet level to help KubeVirt VMs of these network types to properly use DHCP to obtain assigned IP addresses. Kube-OVN supports both DHCPv4 and DHCPv6.</p> <p>The subnet DHCP is configured as follows:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: sn-dual\nspec:\ncidrBlock: \"10.0.0.0/24,240e::a00/120\"\ndefault: false\ndisableGatewayCheck: true\ndisableInterConnection: false\nexcludeIps:\n- 10.0.0.1\n- 240e::a01\ngateway: 10.0.0.1,240e::a01\ngatewayNode: ''\ngatewayType: distributed\nnatOutgoing: false\nprivate: false\nprotocol: Dual\nprovider: ovn\nvpc: vpc-test\nenableDHCP: true\ndhcpV4Options: \"lease_time=3600,router=10.0.0.1,server_id=169.254.0.254,server_mac=00:00:00:2E:2F:B8\"\ndhcpV6Options: \"server_id=00:00:00:2E:2F:C5\"\nenableIPv6RA: true\nipv6RAConfigs: \"address_mode=dhcpv6_stateful,max_interval=30,min_interval=5,send_periodic=true\"\n</code></pre> <ul> <li><code>enableDHCP</code>: Whether to enable the DHCP function for the subnet.</li> <li><code>dhcpV4Options</code>,<code>dhcpV6Options</code>: This field directly exposes DHCP-related options within ovn-nb, please reade DHCP Options for more detail. The default value is  <code>\"lease_time=3600, router=$ipv4_gateway, server_id=169.254.0.254, server_mac=$random_mac\"</code> and <code>server_id=$random_mac</code>\u3002</li> <li><code>enableIPv6RA</code>: Whether to enable the route broadcast function of DHCPv6.</li> <li><code>ipv6RAConfigs</code>\uff1aThis field directly exposes DHCP-related options within ovn-nb Logical_Router_Port, please read Logical Router Port for more detail. The default value is <code>address_mode=dhcpv6_stateful, max_interval=30, min_interval=5, send_periodic=true</code>\u3002</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/dpdk/","title":"DPDK Support","text":"<p>This document describes how Kube-OVN combines with OVS-DPDK to provide a DPDK-type network interface to KubeVirt's virtual machines.</p> <p>Upstream KubeVirt does not currently support OVS-DPDK, users need to use the downstream patch Vhostuser implementation to build KubeVirt by themselves or KVM Device Plugin to use OVS-DPDK.</p>"},{"location":"en/advance/dpdk/#prerequisites","title":"Prerequisites","text":"<ul> <li>The node needs to provide a dedicated NIC for the DPDK driver to run.</li> <li>The node needs to have Hugepages enabled.</li> </ul>"},{"location":"en/advance/dpdk/#set-dpdk-driver","title":"Set DPDK driver","text":"<p>Here we use <code>driverctl</code> for example, please refer to the DPDK documentation for specific parameters and other driver usage:</p> <pre><code>driverctl set-override 0000:00:0b.0 uio_pci_generic\n</code></pre>"},{"location":"en/advance/dpdk/#configure-nodes","title":"Configure Nodes","text":"<p>Labeling OVS-DPDK-enabled nodes for Kube-OVN to recognize:</p> <pre><code>kubectl label nodes &lt;node&gt; ovn.kubernetes.io/ovs_dp_type=\"userspace\"\n</code></pre> <p>Create the configuration file <code>ovs-dpdk-config</code> in the <code>/opt/ovs-config</code> directory on nodes that support DPDK.</p> <pre><code>ENCAP_IP=192.168.122.193/24\nDPDK_DEV=0000:00:0b.0\n</code></pre> <ul> <li><code>ENCAP_IP</code>: The tunnel endpoint address.</li> <li><code>DPDK_DEV</code>: The PCI ID of the device.</li> </ul>"},{"location":"en/advance/dpdk/#install-kube-ovn","title":"Install Kube-OVN","text":"<p>Download scripts:</p> <pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/images/install.sh\n</code></pre> <p>Enable the DPDK installation option:</p> <pre><code>bash install.sh --with-hybrid-dpdk\n</code></pre>"},{"location":"en/advance/dpdk/#usage","title":"Usage","text":"<p>Here we verify the OVS-DPDK functionality by creating a virtual machine with a vhostuser type NIC.</p> <p>Here we use the KVM Device Plugin to create virtual machines. For more information on how to use it, please refer to [KVM Device Plugin].(https://github.com/kubevirt/kubernetes-device-plugins/blob/master/docs/README.kvm.md).</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/kubevirt/kubernetes-device-plugins/master/manifests/kvm-ds.yml\n</code></pre> <p>Create NetworkAttachmentDefinition:</p> <pre><code>apiVersion: k8s.cni.cncf.io/v1\nkind: NetworkAttachmentDefinition\nmetadata:\nname: ovn-dpdk\nnamespace: default\nspec:\nconfig: &gt;-\n{\n\"cniVersion\": \"0.3.0\", \n\"type\": \"kube-ovn\", \n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\", \n\"provider\": \"ovn-dpdk.default.ovn\",\n\"vhost_user_socket_volume_name\": \"vhostuser-sockets\",\n\"vhost_user_socket_name\": \"sock\"\n}\n</code></pre> <p>Create a VM image using the following Dockerfile:</p> <pre><code>FROM quay.io/kubevirt/virt-launcher:v0.46.1\n\n# wget http://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud.qcow2\nCOPY CentOS-7-x86_64-GenericCloud.qcow2 /var/lib/libvirt/images/CentOS-7-x86_64-GenericCloud.qcow2\n</code></pre> <p>Create a virtual machine:</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: vm-config\ndata:\nstart.sh: |\nchmod u+w /etc/libvirt/qemu.conf\necho \"hugetlbfs_mount = \\\"/dev/hugepages\\\"\" &gt;&gt; /etc/libvirt/qemu.conf\nvirtlogd &amp;\nlibvirtd &amp;\n\nmkdir /var/lock\n\nsleep 5\n\nvirsh define /root/vm/vm.xml\nvirsh start vm\n\ntail -f /dev/null\nvm.xml: |\n&lt;domain type='kvm'&gt;\n&lt;name&gt;vm&lt;/name&gt;\n&lt;uuid&gt;4a9b3f53-fa2a-47f3-a757-dd87720d9d1d&lt;/uuid&gt;\n&lt;memory unit='KiB'&gt;2097152&lt;/memory&gt;\n&lt;currentMemory unit='KiB'&gt;2097152&lt;/currentMemory&gt;\n&lt;memoryBacking&gt;\n&lt;hugepages&gt;\n&lt;page size='2' unit='M' nodeset='0'/&gt;\n&lt;/hugepages&gt;\n&lt;/memoryBacking&gt;\n&lt;vcpu placement='static'&gt;2&lt;/vcpu&gt;\n&lt;cputune&gt;\n&lt;shares&gt;4096&lt;/shares&gt;\n&lt;vcpupin vcpu='0' cpuset='4'/&gt;\n&lt;vcpupin vcpu='1' cpuset='5'/&gt;\n&lt;emulatorpin cpuset='1,3'/&gt;\n&lt;/cputune&gt;\n&lt;os&gt;\n&lt;type arch='x86_64' machine='pc'&gt;hvm&lt;/type&gt;\n&lt;boot dev='hd'/&gt;\n&lt;/os&gt;\n&lt;features&gt;\n&lt;acpi/&gt;\n&lt;apic/&gt;\n&lt;/features&gt;\n&lt;cpu mode='host-model'&gt;\n&lt;model fallback='allow'/&gt;\n&lt;topology sockets='1' cores='2' threads='1'/&gt;\n&lt;numa&gt;\n&lt;cell id='0' cpus='0-1' memory='2097152' unit='KiB' memAccess='shared'/&gt;\n&lt;/numa&gt;\n&lt;/cpu&gt;\n&lt;on_reboot&gt;restart&lt;/on_reboot&gt;\n&lt;devices&gt;\n&lt;emulator&gt;/usr/libexec/qemu-kvm&lt;/emulator&gt;\n&lt;disk type='file' device='disk'&gt;\n&lt;driver name='qemu' type='qcow2' cache='none'/&gt;\n&lt;source file='/var/lib/libvirt/images/CentOS-7-x86_64-GenericCloud.qcow2'/&gt;\n&lt;target dev='vda' bus='virtio'/&gt;\n&lt;/disk&gt;\n\n&lt;interface type='vhostuser'&gt;\n&lt;mac address='00:00:00:0A:30:89'/&gt;\n&lt;source type='unix' path='/var/run/vm/sock' mode='server'/&gt;\n&lt;model type='virtio'/&gt;\n&lt;driver queues='2'&gt;\n&lt;host mrg_rxbuf='off'/&gt;\n&lt;/driver&gt;\n&lt;/interface&gt;\n&lt;serial type='pty'&gt;\n&lt;target type='isa-serial' port='0'&gt;\n&lt;model name='isa-serial'/&gt;\n&lt;/target&gt;\n&lt;/serial&gt;\n&lt;console type='pty'&gt;\n&lt;target type='serial' port='0'/&gt;\n&lt;/console&gt;\n&lt;channel type='unix'&gt;\n&lt;source mode='bind' path='/var/lib/libvirt/qemu/channel/target/domain-1-vm/org.qemu.guest_agent.0'/&gt;\n&lt;target type='virtio' name='org.qemu.guest_agent.0' state='connected'/&gt;\n&lt;alias name='channel0'/&gt;\n&lt;address type='virtio-serial' controller='0' bus='0' port='1'/&gt;\n&lt;/channel&gt;\n\n&lt;/devices&gt;\n&lt;/domain&gt;\n---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\nname: vm-deployment\nlabels:\napp: vm\nspec:\nreplicas: 1\nselector:\nmatchLabels:\napp: vm\ntemplate:\nmetadata:\nlabels:\napp: vm\nannotations:\nk8s.v1.cni.cncf.io/networks: default/ovn-dpdk\novn-dpdk.default.ovn.kubernetes.io/ip_address: 10.16.0.96\novn-dpdk.default.ovn.kubernetes.io/mac_address: 00:00:00:0A:30:89\nspec:\nnodeSelector:\novn.kubernetes.io/ovs_dp_type: userspace\nsecurityContext:\nrunAsUser: 0\nvolumes:\n- name: vhostuser-sockets\nemptyDir: {}\n- name: xml\nconfigMap:\nname: vm-config\n- name: hugepage\nemptyDir:\nmedium: HugePages-2Mi\n- name: libvirt-runtime\nemptyDir: {}\ncontainers:\n- name: vm\nimage: vm-vhostuser:latest\ncommand: [\"bash\", \"/root/vm/start.sh\"]\nsecurityContext:\ncapabilities:\nadd:\n- NET_BIND_SERVICE\n- SYS_NICE\n- NET_RAW\n- NET_ADMIN\nprivileged: false\nrunAsUser: 0\nresources:\nlimits:\ncpu: '2'\ndevices.kubevirt.io/kvm: '1'\nmemory: '8784969729'\nhugepages-2Mi: 2Gi\nrequests:\ncpu: 666m\ndevices.kubevirt.io/kvm: '1'\nephemeral-storage: 50M\nmemory: '4490002433'\nvolumeMounts:\n- name: vhostuser-sockets\nmountPath: /var/run/vm\n- name: xml\nmountPath: /root/vm/\n- mountPath: /dev/hugepages\nname: hugepage\n- name: libvirt-runtime\nmountPath: /var/run/libvirt\n</code></pre> <p>Wait for the virtual machine to be created successfully and then go to the Pod to configure the virtual machine:</p> <pre><code># virsh set-user-password vm root 12345\nPassword set successfully for root in vm\n\n# virsh console vm\nConnected to domain 'vm'\nEscape character is ^] (Ctrl + ])\n\nCentOS Linux 7 (Core)\nKernel 3.10.0-1127.el7.x86_64 on an x86_64\n\nlocalhost login: root\nPassword:\nLast login: Fri Feb 25 09:52:54 on ttyS0\n</code></pre> <p>Next, you can log into the virtual machine for network configuration and test:</p> <pre><code>ip link set eth0 mtu 1400\nip addr add 10.16.0.96/16 dev eth0\nip ro add default via 10.16.0.1\nping 114.114.114.114\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/external-gateway/","title":"External Gateway","text":"<p>In some scenarios, all container traffic access to the outside needs to be managed and audited through an external gateway. Kube-OVN can forward outbound traffic to the corresponding external gateway by configuring the appropriate routes in the subnet.</p>"},{"location":"en/advance/external-gateway/#usage","title":"Usage","text":"<pre><code>kind: Subnet\napiVersion: kubeovn.io/v1\nmetadata:\nname: external\nspec:\ncidrBlock: 172.31.0.0/16\ngatewayType: centralized\nnatOutgoing: false\nexternalEgressGateway: 192.168.0.1\npolicyRoutingTableID: 1000\npolicyRoutingPriority: 1500\n</code></pre> <ul> <li><code>natOutgoing</code>: needs to be set to <code>false</code>.</li> <li><code>externalEgressGateway</code>: Set to the address of the external gateway, which needs to be in the same Layer 2 reachable domain as the gateway node.</li> <li><code>policyRoutingTableID</code>: The TableID of the local policy routing table used needs to be different for each subnet to avoid conflicts.</li> <li><code>policyRoutingPriority</code>: Route priority, in order to avoid subsequent user customization of other routing operations conflict, here you can specify the route priority. If no special needs, you can fill in any value.</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/fastpath/","title":"Compile FastPath Module","text":"<p>After a data plane performance profile, <code>netfilter</code> consumes about 20% of CPU resources for related processing within the container and on the host. The FastPath module can bypass <code>netfilter</code> to reduce CPU consumption and latency, and increase throughput. This document will describe how to compile the FastPath module manually.</p>"},{"location":"en/advance/fastpath/#download-related-code","title":"Download Related Code","text":"<pre><code>git clone --depth=1 https://github.com/kubeovn/kube-ovn.git\n</code></pre>"},{"location":"en/advance/fastpath/#install-dependencies","title":"Install Dependencies","text":"<p>Here is an example of CentOS dependencies to download:</p> <pre><code>yum install -y kernel-devel-$(uname -r) gcc elfutils-libelf-devel\n</code></pre>"},{"location":"en/advance/fastpath/#compile-the-module","title":"Compile the Module","text":"<p>For the 3.x kernel:</p> <pre><code>cd kube-ovn/fastpath\nmake all\n</code></pre> <p>For the 4.x kernel:</p> <pre><code>cd kube-ovn/fastpath/4.18\ncp ../Makefile .\nmake all\n</code></pre>"},{"location":"en/advance/fastpath/#instal-the-kernel-module","title":"Instal the Kernel Module","text":"<p>Copy <code>kube_ovn_fastpath.ko</code> to each node that needs performance optimization, and run the following command:</p> <pre><code>insmod kube_ovn_fastpath.ko\n</code></pre> <p>Use <code>dmesg</code> to confirm successful installation:</p> <pre><code># dmesg\n[619631.323788] init_module,kube_ovn_fastpath_local_out\n[619631.323798] init_module,kube_ovn_fastpath_post_routing\n[619631.323800] init_module,kube_ovn_fastpath_pre_routing\n[619631.323801] init_module,kube_ovn_fastpath_local_in\n</code></pre> <p>To uninstall a module, use the following command.</p> <pre><code>rmmod kube_ovn_fastpath.ko\n</code></pre> <p>This module will not be loaded automatically after machine reboot. If you want to load it automatically, please write the corresponding autostart script according to the system configuration.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/multi-nic/","title":"Manage Multiple Interface","text":"<p>Kube-OVN can provide cluster-level IPAM capabilities for other CNI network plugins such as macvlan, vlan, host-device, etc. Other network plugins can then use the subnet and fixed IP capabilities in Kube-OVN.</p> <p>Kube-OVN also supports address management when multiple NICs are all of Kube-OVN type.</p>"},{"location":"en/advance/multi-nic/#working-principle","title":"Working Principle","text":"<p>By using Multus CNI, we can add multiple NICs of different networks to a Pod. However, we still lack the ability to manage the IP addresses of different networks within a cluster. In Kube-OVN, we have been able to perform advanced IP management such as subnet management, IP reservation, random assignment, fixed assignment, etc. through CRD of Subnet and IP. Now Kube-OVN extend the subnet to integrate with other different network plugins, so that other network plugins can also use the IPAM functionality of Kube-OVN.</p>"},{"location":"en/advance/multi-nic/#workflow","title":"Workflow","text":"<p>The above diagram shows how to manage the IP addresses of other network plugins via Kube-OVN. The eth0 NIC of the container is connected to the OVN network and the net1 NIC is connected to other CNI networks. The network definition for the net1 network is taken from the NetworkAttachmentDefinition resource definition in multus-cni.</p> <p>When a Pod is created, <code>kube-ovn-controller</code> will get the Pod add event, find the corresponding Subnet according to the annotation in the Pod, then manage the address from it, and write the address information assigned to the Pod back to the Pod annotation.</p> <p>The CNI on the container machine can configure <code>kube-ovn-cni</code> as the ipam plugin. <code>kube-ovn-cni</code> will read the Pod annotation and return the address information to the corresponding CNI plugin using the standard format of the CNI protocol.</p>"},{"location":"en/advance/multi-nic/#usage","title":"Usage","text":""},{"location":"en/advance/multi-nic/#install-kube-ovn-and-multus","title":"Install Kube-OVN and Multus","text":"<p>Please refer One-Click Installation and Multus how to use to install Kube-OVN and Multus-CNI.</p>"},{"location":"en/advance/multi-nic/#provide-ipam-for-other-types-of-cni","title":"Provide IPAM for other types of CNI","text":""},{"location":"en/advance/multi-nic/#create-networkattachmentdefinition","title":"Create NetworkAttachmentDefinition","text":"<p>Here we use macvlan as the second network of the container network and set its ipam to <code>kube-ovn</code>:</p> <pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: macvlan\nnamespace: default\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.0\",\n\"type\": \"macvlan\",\n\"master\": \"eth0\",\n\"mode\": \"bridge\",\n\"ipam\": {\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"macvlan.default\"\n}\n}'\n</code></pre> <ul> <li><code>spec.config.ipam.type</code>: Need to be set to <code>kube-ovn</code> to call the kube-ovn plugin to get the address information.</li> <li><code>server_socket</code>: The socket file used for communication to Kube-OVN. The default location is <code>/run/openvswitch/kube-ovn-daemon.sock</code>.</li> <li><code>provider</code>: The current NetworkAttachmentDefinition's <code>&lt;name&gt;. &lt;namespace&gt;</code> , Kube-OVN will use this information to find the corresponding Subnet resource.</li> </ul>"},{"location":"en/advance/multi-nic/#the-attached-nic-is-a-kube-ovn-type-nic","title":"The attached NIC is a Kube-OVN type NIC","text":"<p>At this point, the multiple NICs are all Kube-OVN type NICs.</p>"},{"location":"en/advance/multi-nic/#create-networkattachmentdefinition_1","title":"Create NetworkAttachmentDefinition","text":"<p>Set the <code>provider</code> suffix to <code>ovn</code>:</p> <pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: attachnet\nnamespace: default\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.0\",\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"attachnet.default.ovn\"\n}'\n</code></pre> <ul> <li><code>spec.config.ipam.type</code>: Need to be set to <code>kube-ovn</code> to call the kube-ovn plugin to get the address information.</li> <li><code>server_socket</code>: The socket file used for communication to Kube-OVN. The default location is <code>/run/openvswitch/kube-ovn-daemon.sock</code>.</li> <li><code>provider</code>: The current NetworkAttachmentDefinition's <code>&lt;name&gt;. &lt;namespace&gt;</code> , Kube-OVN will use this information to find the corresponding Subnet resource. It should have the suffix <code>ovn</code> here.</li> </ul>"},{"location":"en/advance/multi-nic/#create-a-kube-ovn-subnet","title":"Create a Kube-OVN Subnet","text":"<p>Create a Kube-OVN Subnet, set the corresponding <code>cidrBlock</code> and <code>exclude_ips</code>, the <code>provider</code> should be set to the <code>&lt;name&gt;. &lt;namespace&gt;</code> of corresponding NetworkAttachmentDefinition. For example, to provide additional NICs with macvlan, create a Subnet as follows:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: macvlan\nspec:\nprotocol: IPv4\nprovider: macvlan.default\ncidrBlock: 172.17.0.0/16\ngateway: 172.17.0.1\nexcludeIps:\n- 172.17.0.0..172.17.0.10\n</code></pre> <p><code>gateway</code>, <code>private</code>, <code>nat</code> are only valid for networks with <code>provider</code> type ovn, not for attachment networks.</p> <p>If you are using Kube-OVN as an attached NIC, <code>provider</code> should be set to the <code>&lt;name&gt;. &lt;namespace&gt;.ovn</code> of the corresponding NetworkAttachmentDefinition, and should end with <code>ovn</code> as a suffix.</p> <p>An example of creating a Subnet with an additional NIC provided by Kube-OVN is as follows:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: attachnet\nspec:\nprotocol: IPv4\nprovider: attachnet.default.ovn\ncidrBlock: 172.17.0.0/16\ngateway: 172.17.0.1\nexcludeIps:\n- 172.17.0.0..172.17.0.10\n</code></pre>"},{"location":"en/advance/multi-nic/#create-a-pod-with-multiple-nic","title":"Create a Pod with Multiple NIC","text":"<p>For Pods with randomly assigned addresses, simply add the following annotation <code>k8s.v1.cni.cncf.io/networks</code>, taking the value <code>&lt;namespace&gt;/&lt;name&gt;</code> of the corresponding NetworkAttachmentDefinition.\uff1a</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: samplepod\nnamespace: default\nannotations:\nk8s.v1.cni.cncf.io/networks: default/macvlan\nspec:\ncontainers:\n- name: samplepod\ncommand: [\"/bin/ash\", \"-c\", \"trap : TERM INT; sleep infinity &amp; wait\"]\nimage: docker.io/library/alpine:edge\n</code></pre>"},{"location":"en/advance/multi-nic/#create-pod-with-a-fixed-ip","title":"Create Pod with a Fixed IP","text":"<p>For Pods with fixed IPs, add <code>&lt;networkAttachmentName&gt;.&lt;networkAttachmentNamespace&gt;.kubernetes.io/ip_address</code> annotation\uff1a</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: static-ip\nnamespace: default\nannotations:\nk8s.v1.cni.cncf.io/networks: default/macvlan\novn.kubernetes.io/ip_address: 10.16.0.15\novn.kubernetes.io/mac_address: 00:00:00:53:6B:B6\nmacvlan.default.kubernetes.io/ip_address: 172.17.0.100\nmacvlan.default.kubernetes.io/mac_address: 00:00:00:53:6B:BB\nspec:\ncontainers:\n- name: static-ip\nimage: docker.io/library/nginx:alpine\n</code></pre>"},{"location":"en/advance/multi-nic/#create-workloads-with-fixed-ips","title":"Create Workloads with Fixed IPs","text":"<p>For workloads that use ippool, add <code>&lt;networkAttachmentName&gt;.&lt;networkAttachmentNamespace&gt;.kubernetes.io/ip_pool</code> annotations:</p> <pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nnamespace: default\nname: static-workload\nlabels:\napp: static-workload\nspec:\nreplicas: 2\nselector:\nmatchLabels:\napp: static-workload\ntemplate:\nmetadata:\nlabels:\napp: static-workload\nannotations:\nk8s.v1.cni.cncf.io/networks: default/macvlan\novn.kubernetes.io/ip_pool: 10.16.0.15,10.16.0.16,10.16.0.17\nmacvlan.default.kubernetes.io/ip_pool: 172.17.0.200,172.17.0.201,172.17.0.202\nspec:\ncontainers:\n- name: static-workload\nimage: docker.io/library/nginx:alpine\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/nat-policy-rule/","title":"Default VPC NAT Policy Rule","text":""},{"location":"en/advance/nat-policy-rule/#purpose","title":"Purpose","text":"<p>In the Overlay Subnet under the default VPC, when the <code>natOutgoing</code> switch is turned on, all Pods in the subnet need to do SNAT to access the external network, but in some scenarios we do not want all Pods in the subnet to access the external network by SNAT.</p> <p>So the NAT Policy Rule is to provide a way for users to decide which CIDRs or IPs in the subnet to access the external network need SNAT.</p>"},{"location":"en/advance/nat-policy-rule/#how-to-use-nat-policy-rules","title":"How to use NAT Policy Rules","text":"<p>Enable the <code>natOutgoing</code> switch in <code>subnet.Spec</code>, and add the field <code>natOutgoingPolicyRules</code> as follows:</p> <pre><code>spec:\nnatOutgoing: true\nnatOutgoingPolicyRules:\n- action: forward\nmatch:\nsrcIPs: 10.0.11.0/30,10.0.11.254\n- action: nat\nmatch:\nsrcIPs: 10.0.11.128/26\ndstIPs: 114.114.114.114,8.8.8.8\n</code></pre> <p>The above case shows that there are two NAT policy rules:</p> <ol> <li>Packets with source IP 10.0.11.0/30 or 10.0.11.254 will not perform SNAT when accessing the external network.</li> <li>When a packet with source IP 10.0.11.128/26 and destination IP 114.114.114.114 or 8.8.8.8 accesses the external network, SNAT will be performed.</li> </ol> <p>Field description:</p> <p><code>action</code>: The action that will be executed for packets that meets the corresponding conditions of the <code>match</code>. The action is divided into two types: <code>forward</code> and <code>nat</code>. When natOutgoingPolicyRules is not configured, packets are still SNAT by default.</p> <p><code>match</code>: Indicates the matching segment of the message, the matching segment includes <code>srcIPs</code> and <code>dstIPs</code>, here indicates the source IP and destination IP of the message from the subnet to the external network. <code>match.srcIPs</code> and <code>match.dstIPs</code> support multiple cidr and ip, separated by commas. If multiple match rules overlap, the action that is matched first will be executed according to the order of the <code>natOutgoingPolicyRules</code> array.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/node-local-dns/","title":"NodeLocal DNSCache and Kube-OVN adaptation","text":"<p>NodeLocal DNSCache improves cluster DNS performance by running DNS cache as a DaemonSet on cluster nodes. This function can also be adapted to Kube-OVN.</p>"},{"location":"en/advance/node-local-dns/#nodelocal-dnscache-deployment","title":"Nodelocal DNSCache deployment","text":""},{"location":"en/advance/node-local-dns/#deploy-kubernetes-nodelocal-dnscache","title":"Deploy Kubernetes NodeLocal DNScache","text":"<p>This step refers to Kubernetes official website configuration nodelocaldnscache.</p> <p>Deploy with the following script:</p> <pre><code>#!bin/bash\n\nlocaldns=169.254.20.10\ndomain=cluster.local\nkubedns=10.96.0.10\n\nwget https://raw.githubusercontent.com/kubernetes/kubernetes/master/cluster/addons/dns/nodelocaldns/nodelocaldns.yaml\nsed -i \"s/__PILLAR__LOCAL__DNS__/$localdns/g; s/__PILLAR__DNS__DOMAIN__/$domain/g; s/,__PILLAR__DNS__SERVER__//g; s/__PILLAR__CLUSTER__DNS__/$kubedns/g\" nodelocaldns.yaml\n\nkubectl apply -f nodelocaldns.yaml\n</code></pre> <p>Modify the kubelet configuration file on each node, modify the clusterDNS field in <code>/var/lib/kubelet/config.yaml</code> to the local DNS IP 169.254.20.10, and then restart the kubelet service.</p>"},{"location":"en/advance/node-local-dns/#kube-ovn-corresponding-dns-configuration","title":"Kube-OVN corresponding DNS configuration","text":"<p>After deploying the Nodelocal DNScache component of Kubernetes, Kube-OVN needs to make the following modifications:</p>"},{"location":"en/advance/node-local-dns/#underlay-subnet-enable-u2o-switch","title":"Underlay subnet enable U2O switch","text":"<p>If the underlay subnet needs to use the local DNS function, you need to enable the U2O function, that is, configure <code>spec.u2oInterconnection = true</code> in <code>kubectl edit subnet {your subnet}</code>. If it is an overlay subnet, this step is not required.</p>"},{"location":"en/advance/node-local-dns/#specify-the-corresponding-local-dns-ip-for-kube-ovn-controller","title":"Specify the corresponding local DNS IP for kube-ovn-controller","text":"<pre><code>kubectl edit deployment kube-ovn-controller -n kube-system\n</code></pre> <p>Add field to spec.template.spec.containers.args <code>--node-local-dns-ip=169.254.20.10</code></p>"},{"location":"en/advance/node-local-dns/#rebuild-the-created-pods","title":"Rebuild the created Pods","text":"<p>The reason for this step is to let the Pod regenerate <code>/etc/resolv.conf</code> so that the nameserver points to the local DNS IP. If the nameserver of the Pod is not rebuilt, it will still use the DNS ClusterIP of the cluster. At the same time, if the u2o switch is turned on, the Pod needs to be rebuilt to regenerate the Pod gateway.</p>"},{"location":"en/advance/node-local-dns/#validator-local-dns-cache-function","title":"Validator local DNS cache function","text":"<p>After the above configuration is completed, you can find the Pod verification as follows. You can see that the Pod's DNS server points to the local 169.254.20.10 and successfully resolves the domain name:</p> <pre><code># kubectl exec -it pod1 -- nslookup github.com\nServer:         169.254.20.10\nAddress:        169.254.20.10:53\n\n\nName:   github.com\nAddress: 20.205.243.166\n</code></pre> <p>You can also capture packets at the node and verify as follows. You can see that the DNS query message reaches the local DNS service through the ovn0 network card, and the DNS response message returns in the same way:</p> <pre><code># tcpdump -i any port 53\n\n06:20:00.441889 659246098c56_h P   ifindex 17 00:00:00:73:f1:06 ethertype IPv4 (0x0800), length 75: 10.16.0.2.40230 &gt; 169.254.20.10.53: 1291+ A? baidu.com. (27)\n06:20:00.441889 ovn0  In  ifindex 7 00:00:00:50:32:cd ethertype IPv4 (0x0800), length 75: 10.16.0.2.40230 &gt; 169.254.20.10.53: 1291+ A? baidu.com. (27)\n06:20:00.441950 659246098c56_h P   ifindex 17 00:00:00:73:f1:06 ethertype IPv4 (0x0800), length 75: 10.16.0.2.40230 &gt; 169.254.20.10.53: 1611+ AAAA? baidu.com. (27)\n06:20:00.441950 ovn0  In  ifindex 7 00:00:00:50:32:cd ethertype IPv4 (0x0800), length 75: 10.16.0.2.40230 &gt; 169.254.20.10.53: 1611+ AAAA? baidu.com. (27)\n06:20:00.442203 ovn0  Out ifindex 7 00:00:00:52:99:d8 ethertype IPv4 (0x0800), length 145: 169.254.20.10.53 &gt; 10.16.0.2.40230: 1611* 0/1/0 (97)\n06:20:00.442219 659246098c56_h Out ifindex 17 00:00:00:ea:b3:5e ethertype IPv4 (0x0800), length 145: 169.254.20.10.53 &gt; 10.16.0.2.40230: 1611* 0/1/0 (97)\n06:20:00.442273 ovn0  Out ifindex 7 00:00:00:52:99:d8 ethertype IPv4 (0x0800), length 125: 169.254.20.10.53 &gt; 10.16.0.2.40230: 1291* 2/0/0 A 39.156.66.10, A 110.242.68.66 (77)\n06:20:00.442278 659246098c56_h Out ifindex 17 00:00:00:ea:b3:5e ethertype IPv4 (0x0800), length 125: 169.254.20.10.53 &gt; 10.16.0.2.40230: 1291* 2/0/0 A 39.156.66.10, A 110.242.68.66 (77)\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/offload-corigine/","title":"Offload with Corigine","text":"<p>Kube-OVN uses OVS for traffic forwarding in the final data plane, and the associated flow table matching, tunnel encapsulation and other functions are CPU-intensive, which consumes a lot of CPU resources and leads to higher latency and lower throughput under heavy traffic. Corigine Agilio CX series SmartNIC can offload OVS-related operations to the hardware. This technology can shorten the data path without modifying the OVS control plane, avoiding the use of host CPU resources, which dramatically reduce latency and significantly increase the throughput.</p> <p></p>"},{"location":"en/advance/offload-corigine/#prerequisites","title":"Prerequisites","text":"<ul> <li>Corigine Agilio CX series SmartNIC.</li> <li>CentOS 8 Stream or Linux 5.7 above.</li> <li>Since the current NIC does not support <code>dp_hash</code> and <code>hash</code> operation offload, OVN LB function should be disabled.</li> </ul>"},{"location":"en/advance/offload-corigine/#setup-sr-iov","title":"Setup SR-IOV","text":"<p>Please read Agilio Open vSwitch TC User Guide for the detail usage of this SmartNIC.</p> <p>The following scripts are saved for subsequent execution of firmware-related operations:</p> <pre><code>#!/bin/bash\nDEVICE=${1}\nDEFAULT_ASSY=scan\nASSY=${2:-${DEFAULT_ASSY}}\nAPP=${3:-flower}\n\nif [ \"x${DEVICE}\" = \"x\" -o ! -e /sys/class/net/${DEVICE} ]; then\necho Syntax: ${0} device [ASSY] [APP]\necho\necho This script associates the TC Offload firmware\n    echo with a Netronome SmartNIC.\n    echo\necho device: is the network device associated with the SmartNIC\n    echo ASSY: defaults to ${DEFAULT_ASSY}\necho APP: defaults to flower. flower-next is supported if updated\n    echo      firmware has been installed.\n    exit 1\nfi\n\n# It is recommended that the assembly be determined by inspection\n# The following code determines the value via the debug interface\nif [ \"${ASSY}x\" = \"scanx\" ]; then\nethtool -W ${DEVICE} 0\nDEBUG=$(ethtool -w ${DEVICE} data /dev/stdout | strings)\nSERIAL=$(echo \"${DEBUG}\" | grep \"^SN:\")\nASSY=$(echo ${SERIAL} | grep -oE AMDA[0-9]{4})\nfi\n\nPCIADDR=$(basename $(readlink -e /sys/class/net/${DEVICE}/device))\nFWDIR=\"/lib/firmware/netronome\"\n\n# AMDA0081 and AMDA0097 uses the same firmware\nif [ \"${ASSY}\" = \"AMDA0081\" ]; then\nif [ ! -e ${FWDIR}/${APP}/nic_AMDA0081.nffw ]; then\nln -sf nic_AMDA0097.nffw ${FWDIR}/${APP}/nic_AMDA0081.nffw\n   fi\nfi\n\nFW=\"${FWDIR}/pci-${PCIADDR}.nffw\"\nln -sf \"${APP}/nic_${ASSY}.nffw\" \"${FW}\"\n\n# insert distro-specific initramfs section here...\n</code></pre> <p>Switching firmware options and reloading the driver:</p> <pre><code>./agilio-tc-fw-select.sh ens47np0 scan\nrmmod nfp\nmodprobe nfp\n</code></pre> <p>Check the number of available VFs and create VFs.</p> <pre><code># cat /sys/class/net/ens3/device/sriov_totalvfs\n65\n\n# echo 4 &gt; /sys/class/net/ens47/device/sriov_numvfs\n</code></pre>"},{"location":"en/advance/offload-corigine/#install-sr-iov-device-plugin","title":"Install SR-IOV Device Plugin","text":"<p>Since each machine has a limited number of VFs and each Pod that uses acceleration will take up VF resources, we need to use the SR-IOV Device Plugin to manage the corresponding resources so that the scheduler knows how to schedule.</p> <p>Create SR-IOV Configmap:</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: sriovdp-config\nnamespace: kube-system\ndata:\nconfig.json: |\n{\n\"resourceList\": [{\n\"resourcePrefix\": \"corigine.com\",\n\"resourceName\": \"agilio_sriov\",\n\"selectors\": {\n\"vendors\": [\"19ee\"],\n\"devices\": [\"6003\"],\n\"drivers\": [\"nfp_netvf\"]\n}\n}\n]\n}\n</code></pre> <p>Please read the SR-IOV device plugin to deploy:</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/intel/sriov-network-device-plugin/master/deployments/k8s-v1.16/sriovdp-daemonset.yaml\n</code></pre> <p>Check if SR-IOV resources have been registered to Kubernetes Node:</p> <pre><code>kubectl describe no containerserver  | grep corigine\n\ncorigine.com/agilio_sriov:  4\ncorigine.com/agilio_sriov:  4\ncorigine.com/agilio_sriov  0           0\n</code></pre>"},{"location":"en/advance/offload-corigine/#install-multus-cni","title":"Install Multus-CNI","text":"<p>The device IDs obtained during SR-IOV Device Plugin scheduling need to be passed to Kube-OVN via Multus-CNI, so Multus-CNI needs to be configured to perform the related tasks.</p> <p>Please read Multus-CNI Document to deploy\uff1a</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml\n</code></pre> <p>Create <code>NetworkAttachmentDefinition</code>\uff1a</p> <pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: default\nnamespace: default\nannotations:\nk8s.v1.cni.cncf.io/resourceName: corigine.com/agilio_sriov\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.1\",\n\"name\": \"kube-ovn\",\n\"plugins\":[\n{\n\"type\":\"kube-ovn\",\n\"server_socket\":\"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"default.default.ovn\"\n},\n{\n\"type\":\"portmap\",\n\"capabilities\":{\n\"portMappings\":true\n}\n}\n]\n}'\n</code></pre> <ul> <li><code>provider</code>: the format should be {name}.{namespace}.ovn of related <code>NetworkAttachmentDefinition</code>.</li> </ul>"},{"location":"en/advance/offload-corigine/#enable-offload-in-kube-ovn","title":"Enable Offload in Kube-OVN","text":"<p>Download the scripts:</p> <pre><code>wget https://raw.githubusercontent.com/alauda/kube-ovn/master/dist/images/install.sh\n</code></pre> <p>Change the related options\uff0c<code>IFACE</code> should be the physic NIC and has an IP:</p> <pre><code>ENABLE_MIRROR=${ENABLE_MIRROR:-false}\nHW_OFFLOAD=${HW_OFFLOAD:-true}\nENABLE_LB=${ENABLE_LB:-false}\nIFACE=\"ensp01\"\n</code></pre> <p>Install Kube-OVN\uff1a</p> <pre><code>bash install.sh\n</code></pre>"},{"location":"en/advance/offload-corigine/#create-pods-with-vf-nics","title":"Create Pods with VF NICs","text":"<p>Pods that use VF for network offload acceleration can be created using the following yaml:</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: nginx\nnamespace: default\nannotations:\nv1.multus-cni.io/default-network: default/default\nspec:\ncontainers:\n- name: nginx\nimage: docker.io/library/nginx:alpine\nresources:\nrequests:\ncorigine.com/agilio_sriov: '1'\nlimits:\ncorigine.com/agilio_sriov: '1'\n</code></pre> <ul> <li><code>v1.multus-cni.io/default-network</code>: should be the {namespace}/{name} of related <code>NetworkAttachmentDefinition</code>.</li> </ul> <p>Running the following command in the <code>ovs-ovn</code> container of the Pod run node to observe if offload success.</p> <pre><code># ovs-appctl dpctl/dump-flows -m type=offloaded\nufid:91cc45de-e7e9-4935-8f82-1890430b0f66, skb_priority(0/0),skb_mark(0/0),ct_state(0/0x23),ct_zone(0/0),ct_mark(0/0),ct_label(0/0x1),recirc_id(0),dp_hash(0/0),in_port(5b45c61b307e_h),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:c5:6d:4e,dst=00:00:00:e7:16:ce),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:941539, bytes:62142230, used:0.260s, offloaded:yes, dp:tc, actions:54235e5753b8_h\nufid:e00768d7-e652-4d79-8182-3291d852b791, skb_priority(0/0),skb_mark(0/0),ct_state(0/0x23),ct_zone(0/0),ct_mark(0/0),ct_label(0/0x1),recirc_id(0),dp_hash(0/0),in_port(54235e5753b8_h),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:e7:16:ce,dst=00:00:00:c5:6d:4e),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:82386659, bytes:115944854173, used:0.260s, offloaded:yes, dp:tc, actions:5b45c61b307e_h\n</code></pre> <p>If there is <code>offloaded:yes, dp:tc</code> content, the offloading is successful.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/offload-mellanox/","title":"Offload with Mellanox","text":"<p>Kube-OVN uses OVS for traffic forwarding in the final data plane, and the associated flow table matching, tunnel encapsulation and other functions are CPU-intensive, which consumes a lot of CPU resources and leads to higher latency and lower throughput under heavy traffic. Mellanox Accelerated Switching And Packet Processing (ASAP\u00b2) technology offloads OVS-related operations to an eSwitch within the eSwitch in the hardware. This technology can shorten the data path without modifying the OVS control plane, avoiding the use of host CPU resources, which dramatically reduce latency and significantly increase the throughput.</p> <p></p>"},{"location":"en/advance/offload-mellanox/#prerequisites","title":"Prerequisites","text":"<ul> <li>Mellanox CX5/CX6/BlueField that support ASAP\u00b2.</li> <li>CentOS 8 Stream or Linux 5.7 above.</li> <li>Since the current NIC does not support <code>dp_hash</code> and <code>hash</code> operation offload, OVN LB function should be disabled.</li> <li>In order to support offload mode, the NIC cannot do bond.</li> </ul>"},{"location":"en/advance/offload-mellanox/#setup-sr-iov","title":"Setup SR-IOV","text":"<p>Check the device ID of the NIC, in the following example it is <code>42:00.0</code>:</p> <pre><code># lspci -nn | grep ConnectX-5\n42:00.0 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5] [15b3:1017]\n</code></pre> <p>Find the corresponding NIC by its device ID:</p> <pre><code># ls -l /sys/class/net/ | grep 42:00.0\nlrwxrwxrwx. 1 root root 0 Jul 22 23:16 p4p1 -&gt; ../../devices/pci0000:40/0000:40:02.0/0000:42:00.0/net/p4p1\n</code></pre> <p>Check the number of available VFs:</p> <pre><code># cat /sys/class/net/p4p1/device/sriov_totalvfs\n8\n</code></pre> <p>Create VFs and do not exceeding the number found above:</p> <pre><code># echo '4' &gt; /sys/class/net/p4p1/device/sriov_numvfs\n# ip link show p4p1\n10: p4p1: &lt;NO-CARRIER,BROADCAST,MULTICAST,UP&gt; mtu 1500 qdisc mq state DOWN mode DEFAULT group default qlen 1000\nlink/ether b8:59:9f:c1:ec:12 brd ff:ff:ff:ff:ff:ff\n    vf 0 MAC 00:00:00:00:00:00, spoof checking off, link-state auto, trust off, query_rss off\n    vf 1 MAC 00:00:00:00:00:00, spoof checking off, link-state auto, trust off, query_rss off\n    vf 2 MAC 00:00:00:00:00:00, spoof checking off, link-state auto, trust off, query_rss off\n    vf 3 MAC 00:00:00:00:00:00, spoof checking off, link-state auto, trust off, query_rss off\n# ip link set p4p1 up\n</code></pre> <p>Find the device IDs corresponding to the above VFs:</p> <pre><code># lspci -nn | grep ConnectX-5\n42:00.0 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5] [15b3:1017]\n42:00.1 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5] [15b3:1017]\n42:00.2 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n42:00.3 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n42:00.4 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n42:00.5 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n</code></pre> <p>Unbound the VFs from the driver:</p> <pre><code>echo 0000:42:00.2 &gt; /sys/bus/pci/drivers/mlx5_core/unbind\necho 0000:42:00.3 &gt; /sys/bus/pci/drivers/mlx5_core/unbind\necho 0000:42:00.4 &gt; /sys/bus/pci/drivers/mlx5_core/unbind\necho 0000:42:00.5 &gt; /sys/bus/pci/drivers/mlx5_core/unbind\n</code></pre> <p>Enable eSwitch mode and set up hardware offload:</p> <pre><code>devlink dev eswitch set pci/0000:42:00.0 mode switchdev\nethtool -K enp66s0f0 hw-tc-offload on\n</code></pre> <p>Rebind the driver and complete the VF setup:</p> <pre><code>echo 0000:42:00.2 &gt; /sys/bus/pci/drivers/mlx5_core/bind\necho 0000:42:00.3 &gt; /sys/bus/pci/drivers/mlx5_core/bind\necho 0000:42:00.4 &gt; /sys/bus/pci/drivers/mlx5_core/bind\necho 0000:42:00.5 &gt; /sys/bus/pci/drivers/mlx5_core/bind\n</code></pre> <p>Some behaviors of <code>NetworkManager</code> may cause driver exceptions, if offloading problems occur we recommended to close <code>NetworkManager</code> and try again.</p> <pre><code>systemctl stop NetworkManager\nsystemctl disable NetworkManager\n</code></pre>"},{"location":"en/advance/offload-mellanox/#install-sr-iov-device-plugin","title":"Install SR-IOV Device Plugin","text":"<p>Since each machine has a limited number of VFs and each Pod that uses acceleration will take up VF resources, we need to use the SR-IOV Device Plugin to manage the corresponding resources so that the scheduler knows how to schedule.</p> <p>Create SR-IOV Configmap:</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: sriovdp-config\nnamespace: kube-system\ndata:\nconfig.json: |\n{\n\"resourceList\": [{\n\"resourcePrefix\": \"mellanox.com\",\n\"resourceName\": \"cx5_sriov_switchdev\",\n\"selectors\": {\n\"vendors\": [\"15b3\"],\n\"devices\": [\"1018\"],\n\"drivers\": [\"mlx5_core\"]\n}\n}\n]\n}\n</code></pre> <p>Please read the SR-IOV device plugin to deploy:</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/intel/sriov-network-device-plugin/master/deployments/k8s-v1.16/sriovdp-daemonset.yaml\n</code></pre> <p>Check if SR-IOV resources have been registered to Kubernetes Node:</p> <pre><code>kubectl describe node kube-ovn-01  | grep mellanox\n\nmellanox.com/cx5_sriov_switchdev:  4\nmellanox.com/cx5_sriov_switchdev:  4\nmellanox.com/cx5_sriov_switchdev  0           0\n</code></pre>"},{"location":"en/advance/offload-mellanox/#install-multus-cni","title":"Install Multus-CNI","text":"<p>The device IDs obtained during SR-IOV Device Plugin scheduling need to be passed to Kube-OVN via Multus-CNI, so Multus-CNI needs to be configured to perform the related tasks.</p> <p>Please read Multus-CNI Document to deploy\uff1a</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml\n</code></pre> <p>Create <code>NetworkAttachmentDefinition</code>\uff1a</p> <pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: default\nnamespace: default\nannotations:\nk8s.v1.cni.cncf.io/resourceName: mellanox.com/cx5_sriov_switchdev\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.1\",\n\"name\": \"kube-ovn\",\n\"plugins\":[\n{\n\"type\":\"kube-ovn\",\n\"server_socket\":\"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"default.default.ovn\"\n},\n{\n\"type\":\"portmap\",\n\"capabilities\":{\n\"portMappings\":true\n}\n}\n]\n}'\n</code></pre> <ul> <li><code>provider</code>: the format should be {name}.{namespace}.ovn of related <code>NetworkAttachmentDefinition</code>.</li> </ul>"},{"location":"en/advance/offload-mellanox/#enable-offload-in-kube-ovn","title":"Enable Offload in Kube-OVN","text":"<p>Download the scripts:</p> <pre><code>wget https://raw.githubusercontent.com/alauda/kube-ovn/master/dist/images/install.sh\n</code></pre> <p>Change the related options\uff0c<code>IFACE</code> should be the physic NIC and has an IP:</p> <pre><code>ENABLE_MIRROR=${ENABLE_MIRROR:-false}\nHW_OFFLOAD=${HW_OFFLOAD:-true}\nENABLE_LB=${ENABLE_LB:-false}\nIFACE=\"ensp01\"\n</code></pre> <p>Install Kube-OVN\uff1a</p> <pre><code>bash install.sh\n</code></pre>"},{"location":"en/advance/offload-mellanox/#create-pods-with-vf-nics","title":"Create Pods with VF NICs","text":"<p>Pods that use VF for network offload acceleration can be created using the following yaml:</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: nginx\nannotations:\nv1.multus-cni.io/default-network: default/default\nspec:\ncontainers:\n- name: nginx\nimage: docker.io/library/nginx:alpine\nresources:\nrequests:\nmellanox.com/cx5_sriov_switchdev: '1'\nlimits:\nmellanox.com/cx5_sriov_switchdev: '1'\n</code></pre> <ul> <li><code>v1.multus-cni.io/default-network</code>: should be the {namespace}/{name} of related <code>NetworkAttachmentDefinition</code>.</li> </ul> <p>Running the following command in the <code>ovs-ovn</code> container of the Pod run node to observe if offload success.</p> <pre><code># ovs-appctl dpctl/dump-flows -m type=offloaded\nufid:91cc45de-e7e9-4935-8f82-1890430b0f66, skb_priority(0/0),skb_mark(0/0),ct_state(0/0x23),ct_zone(0/0),ct_mark(0/0),ct_label(0/0x1),recirc_id(0),dp_hash(0/0),in_port(5b45c61b307e_h),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:c5:6d:4e,dst=00:00:00:e7:16:ce),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:941539, bytes:62142230, used:0.260s, offloaded:yes, dp:tc, actions:54235e5753b8_h\nufid:e00768d7-e652-4d79-8182-3291d852b791, skb_priority(0/0),skb_mark(0/0),ct_state(0/0x23),ct_zone(0/0),ct_mark(0/0),ct_label(0/0x1),recirc_id(0),dp_hash(0/0),in_port(54235e5753b8_h),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:e7:16:ce,dst=00:00:00:c5:6d:4e),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:82386659, bytes:115944854173, used:0.260s, offloaded:yes, dp:tc, actions:5b45c61b307e_h\n</code></pre> <p>If there is <code>offloaded:yes, dp:tc</code> content, the offloading is successful.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/overlay-with-route/","title":"Interconnection with Routes in Overlay Mode","text":"<p>In some scenarios, the network environment does not support Underlay mode, but still need Pods and external devices directly access through IP, then you can use the routing method to connect the container network and the external.</p> <p>Only Overlay Subnets in default VPC support this method. In this case, the Pod IP goes directly to the underlying network, which needs to disable IP checks for source and destination addresses.</p>"},{"location":"en/advance/overlay-with-route/#prerequisites","title":"Prerequisites","text":"<ul> <li>In this mode, the host needs to open the <code>ip_forward</code>.</li> <li>Check if there is a <code>Drop</code> rule in the forward chain in the host iptables that should be modified for container-related traffic.</li> <li>Due to the possibility of asymmetric routing, the host needs to allow packets with a ct status of <code>INVALID</code>.</li> </ul>"},{"location":"en/advance/overlay-with-route/#steps","title":"Steps","text":"<p>For subnets that require direct external routing, you need to set <code>natOutgoing</code> of the subnet to <code>false</code> to turn off nat mapping and make the Pod IP directly accessible to the external network.</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: routed\nspec:\nprotocol: IPv4\ncidrBlock: 10.166.0.0/16\ndefault: false\nexcludeIps:\n- 10.166.0.1\ngateway: 10.166.0.1\ngatewayType: distributed\nnatOutgoing: false\n</code></pre> <p>At this point, the Pod's packets can reach the peer node via the host route, but the peer node does not yet know where the return packets should be sent to and needs to add a return route.</p> <p>If the peer host and the container host are on the same Layer 2 network, we can add a static route directly to the peer host to point the next hop of the container network to any machine in the Kubernetes cluster.</p> <pre><code>ip route add 10.166.0.0/16 via 192.168.2.10 dev eth0\n</code></pre> <p><code>10.166.0.0/16</code> is the container subnet CIDR, and <code>192.168.2.10</code> is one node in the Kubernetes cluster.</p> <p>If the peer host and the container host are not in the same layer 2 network, you need to configure the corresponding rules on the router.</p> <p>Note: Specifying an IP for a single node may lead to single point of failure. To achieve fast failover, Keepalived can be used to set up a VIP for multiple nodes, and the next hop of the route can be directed to the VIP.</p> <p>In some virtualized environments, the virtual network identifies asymmetric traffic as illegal traffic and drops it. In this case, you need to adjust the <code>gatewayType</code> of the Subnet to <code>centralized</code> and set the next hop to the IP of the <code>gatewayNode</code> node during route setup.</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: routed\nspec:\nprotocol: IPv4\ncidrBlock: 10.166.0.0/16\ndefault: false\nexcludeIps:\n- 10.166.0.1\ngateway: 10.166.0.1\ngatewayType: centralized\ngatewayNode: \"node1\"\nnatOutgoing: false\n</code></pre> <p>If you still want to perform NAT processing for some traffic, such as traffic accessing the Internet, please refer to the Default VPC NAT Policy Rule.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/ovn-eip-fip-snat/","title":"Support OVN EIP,FIP and SNAT","text":"<pre><code>graph LR\n\n\npod--&gt;vpc1-subnet--&gt;vpc1--&gt;snat--&gt;lrp--&gt;external-subnet--&gt;gw-node-external-nic</code></pre> <p>The pod access the public network based on the snat</p> <pre><code>graph LR\n\n\npod--&gt;vpc1-subnet--&gt;vpc1--&gt;fip--&gt;lrp--&gt;external-subnet--&gt;local-node-external-nic</code></pre> <p>The pod access the public network based on the fip</p> <p>The CRD supported by this function is basically the same as the iptable nat gw public network solution.</p> <ul> <li>ovn eip: occupies a public ip address and is allocated from the underlay provider network vlan subnet</li> <li>ovn fip: one-to-one dnat snat, which provides direct public network access for ip addresses and vip in a vpc</li> <li>ovn snat: a subnet cidr or a single vpc ip or vip can access public networks based on snat</li> <li>ovn dnat: based router lb, which enables direct access to a group of endpoints in a vpc based on a public endpoint</li> </ul>"},{"location":"en/advance/ovn-eip-fip-snat/#1-deployment","title":"1. Deployment","text":"<p>Currently allows all vpcs to share the same provider vlan subnet resources, similar to neutron ovn mode. Compatible with previous scenarios default VPC EIP/SNAT.</p> <p>During the deployment phase, you may need to specify a default public network logical switch based on actual conditions. If no vlan is in use (vlan 0 is used), the following startup parameters do not need to be configured.</p> <pre><code># When deploying you need to refer to the above scenario and specify the following parameters as needed according to the actual situation\n# 1. kube-ovn-controller Startup parameters to be configured\uff1a\n- --external-gateway-vlanid=204\n- --external-gateway-switch=external204\n\n# 2. kube-ovn-cni Startup parameters to be configured:\n- --external-gateway-switch=external204 # The above configuration is consistent with the following public network configuration vlan id and resource name, \n# currently only support to specify one underlay public network as the default external public network.\n</code></pre> <p>The design and use of this configuration item takes into account the following factors\uff1a</p> <ul> <li>Based on this configuration item can be docked to the provider network, vlan, subnet resources.</li> <li>Based on this configuration item, the default vpc enable_eip_snat function can be docked to the existing vlan, subnet resources, while supporting the ipam</li> <li>If only the default vpc's enable_eip_snat mode is used with the old pod annotaion based eip fip snat, then the following configuration is not required.</li> <li>Based on this configuration you can not use the default vpc enable_eip_snat process, only by corresponding to vlan, subnet process, can be compatible with only custom vpc use eip snat usage scenarios.</li> </ul> <p>The neutron ovn mode also has a certain static file configuration designation that is, for now, generally consistent.</p>"},{"location":"en/advance/ovn-eip-fip-snat/#11-create-the-underlay-public-network","title":"1.1 Create the underlay public network","text":"<pre><code># provider-network\uff0c vlan\uff0c subnet\n# cat 01-provider-network.yaml\napiVersion: kubeovn.io/v1\nkind: ProviderNetwork\nmetadata:\n  name: external204\nspec:\n  defaultInterface: vlan\n\n# cat 02-vlan.yaml\napiVersion: kubeovn.io/v1\nkind: Vlan\nmetadata:\n  name: vlan204\nspec:\n  id: 204\nprovider: external204\n\n# cat 03-vlan-subnet.yaml\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n  name: external204\nspec:\n  protocol: IPv4\n  cidrBlock: 10.5.204.0/24\n  gateway: 10.5.204.254\n  vlan: vlan204\n  excludeIps:\n  - 10.5.204.1..10.5.204.100\n</code></pre>"},{"location":"en/advance/ovn-eip-fip-snat/#12-default-vpc-enable-eip_snat","title":"1.2 Default vpc enable eip_snat","text":"<pre><code># Enable the default vpc and the above underlay public provider subnet interconnection\ncat 00-centralized-external-gw-no-ip.yaml\napiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: ovn-external-gw-config\n  namespace: kube-system\ndata:\n  enable-external-gw: \"true\"\nexternal-gw-nodes: \"pc-node-1,pc-node-2,pc-node-3\" type: \"centralized\"  external-gw-nic: \"vlan\"\nexternal-gw-addr: \"10.5.204.254/24\"\n</code></pre> <p>This feature currently supports the ability to create lrp type ovn eip resources without specifying the lrp ip and mac, which is already supported for automatic acquisition. If specified, it is equivalent to specifying the ip to create an ovn-eip of type lrp. Of course, you can also manually create the lrp type ovn eip in advance.</p>"},{"location":"en/advance/ovn-eip-fip-snat/#13-custom-vpc-enable-eip-snat-fip-function","title":"1.3 Custom vpc enable eip snat fip function","text":"<pre><code># cat 00-ns.yml\napiVersion: v1\nkind: Namespace\nmetadata:\n  name: vpc1\n\n# cat 01-vpc-ecmp-enable-external-bfd.yml\nkind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\n  name: vpc1\nspec:\n  namespaces:\n  - vpc1\n  enableExternal: true\n# vpc enableExternal will automatically create an lrp association to the public network specified above\n\n# cat 02-subnet.yml\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n  name: vpc1-subnet1\nspec:\n  cidrBlock: 192.168.0.0/24\n  default: false\ndisableGatewayCheck: false\ndisableInterConnection: true\nenableEcmp: true\ngatewayNode: \"\"\ngatewayType: distributed\n  #gatewayType: centralized\nnatOutgoing: false\nprivate: false\nprotocol: IPv4\n  provider: ovn\n  vpc: vpc1\n  namespaces:\n  - vpc1\n</code></pre> <p>After the above template is applied, you should see the following resources exist</p> <pre><code># k ko nbctl show vpc1\nrouter 87ad06fd-71d5-4ff8-a1f0-54fa3bba1a7f (vpc1)\nport vpc1-vpc1-subnet1\n        mac: \"00:00:00:ED:8E:C7\"\nnetworks: [\"192.168.0.1/24\"]\nport vpc1-external204\n        mac: \"00:00:00:EF:05:C7\"\nnetworks: [\"10.5.204.105/24\"]\ngateway chassis: [7cedd14f-265b-42e5-ac17-e03e7a1f2342 276baccb-fe9c-4476-b41d-05872a94976d fd9f140c-c45d-43db-a6c0-0d4f8ea298dd]\nnat 21d853b0-f7b4-40bd-9a53-31d2e2745739\n        external ip: \"10.5.204.115\"\nlogical ip: \"192.168.0.0/24\"\ntype: \"snat\"\n</code></pre> <pre><code># k ko nbctl lr-route-list vpc1\nIPv4 Routes\nRoute Table &lt;main&gt;:\n                0.0.0.0/0              10.5.204.254 dst-ip\n# The route currently supports automatic maintenance\n</code></pre>"},{"location":"en/advance/ovn-eip-fip-snat/#2-ovn-eip","title":"2. ovn-eip","text":"<p>This function is designed and used in the same way as iptables-eip, ovn-eip currently has three types</p> <ul> <li>nat: indicates ovn dnat, fip, and snat. These nat types are recorded in status</li> <li>lrp: indicates the resource used to connect a vpc to the public network</li> <li>lsp: In the ovn BFD-based ecmp static route scenario, an ovs internal port is provided on the gateway node as the next hop of the ecmp route</li> </ul> <pre><code>---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: eip-static\nspec:\n  externalSubnet: external204\n  type: nat\n\n# Dynamically allocate an eip resource that is reserved for fip dnat_and_snat scenarios\n</code></pre>"},{"location":"en/advance/ovn-eip-fip-snat/#21-create-an-fip-for-pod","title":"2.1 Create an fip for pod","text":"<pre><code># k get po -o wide -n vpc1 vpc-1-busybox01\nNAME              READY   STATUS    RESTARTS   AGE     IP            NODE\nvpc-1-busybox01   1/1     Running   0          3d15h   192.168.0.2   pc-node-2\n\n# k get ip vpc-1-busybox01.vpc1\nNAME                   V4IP          V6IP   MAC                 NODE        SUBNET\nvpc-1-busybox01.vpc1   192.168.0.2          00:00:00:0A:DD:27   pc-node-2   vpc1-subnet1\n\n---\n\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: eip-static\nspec:\n  externalSubnet: external204\n  type: nat\n\n---\nkind: OvnFip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: eip-static\nspec:\n  ovnEip: eip-static\n  ipName: vpc-1-busybox01.vpc1  # the name of the ip crd, which is unique\n</code></pre> <pre><code># k get ofip\nNAME          VPC    V4EIP          V4IP          READY   IPTYPE   IPNAME\neip-for-vip   vpc1   10.5.204.106   192.168.0.3   true    vip      test-fip-vip\neip-static    vpc1   10.5.204.101   192.168.0.2   true             vpc-1-busybox01.vpc1\n# k get ofip eip-static\nNAME         VPC    V4EIP          V4IP          READY   IPTYPE   IPNAME\neip-static   vpc1   10.5.204.101   192.168.0.2   true             vpc-1-busybox01.vpc1\n\n[root@pc-node-1 03-cust-vpc]# ping 10.5.204.101\nPING 10.5.204.101 (10.5.204.101) 56(84) bytes of data.\n64 bytes from 10.5.204.101: icmp_seq=2 ttl=62 time=1.21 ms\n64 bytes from 10.5.204.101: icmp_seq=3 ttl=62 time=0.624 ms\n64 bytes from 10.5.204.101: icmp_seq=4 ttl=62 time=0.368 ms\n^C\n--- 10.5.204.101 ping statistics ---\n4 packets transmitted, 3 received, 25% packet loss, time 3049ms\nrtt min/avg/max/mdev = 0.368/0.734/1.210/0.352 ms\n[root@pc-node-1 03-cust-vpc]#\n\n# pod &lt;--&gt; node ping is working\n</code></pre> <pre><code># The key resources that this public ip can pass include the following ovn nb resources\n\n# k ko nbctl show vpc1\nrouter 87ad06fd-71d5-4ff8-a1f0-54fa3bba1a7f (vpc1)\nport vpc1-vpc1-subnet1\n        mac: \"00:00:00:ED:8E:C7\"\nnetworks: [\"192.168.0.1/24\"]\nport vpc1-external204\n        mac: \"00:00:00:EF:05:C7\"\nnetworks: [\"10.5.204.105/24\"]\ngateway chassis: [7cedd14f-265b-42e5-ac17-e03e7a1f2342 276baccb-fe9c-4476-b41d-05872a94976d fd9f140c-c45d-43db-a6c0-0d4f8ea298dd]\nnat 813523e7-c68c-408f-bd8c-cba30cb2e4f4\n        external ip: \"10.5.204.101\"\nlogical ip: \"192.168.0.2\"\ntype: \"dnat_and_snat\"\n</code></pre>"},{"location":"en/advance/ovn-eip-fip-snat/#22-create-an-fip-for-vip","title":"2.2 Create an fip for vip","text":"<p>In order to facilitate the use of some vip scenarios, such as inside kubevirt VM, keepalived use vip, kube-vip use vip, etc. the vip need public network access.</p> <pre><code># First create vip, eip, then bind eip to vip\n# cat vip.yaml\napiVersion: kubeovn.io/v1\nkind: Vip\nmetadata:\n  name: test-fip-vip\nspec:\n  subnet: vpc1-subnet1\n\n# cat 04-fip.yaml\n---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: eip-for-vip\nspec:\n  externalSubnet: external204\n  type: nat\n\n---\nkind: OvnFip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: eip-for-vip\nspec:\n  ovnEip: eip-for-vip\n  ipType: vip         # By default fip is for pod ip, here you need to specify the docking to vip resources\nipName: test-fip-vip\n</code></pre> <pre><code># k get ofip\nNAME          VPC    V4EIP          V4IP          READY   IPTYPE   IPNAME\neip-for-vip   vpc1   10.5.204.106   192.168.0.3   true    vip      test-fip-vip\n\n\n[root@pc-node-1 fip-vip]# ping  10.5.204.106\nPING 10.5.204.106 (10.5.204.106) 56(84) bytes of data.\n64 bytes from 10.5.204.106: icmp_seq=1 ttl=62 time=0.694 ms\n64 bytes from 10.5.204.106: icmp_seq=2 ttl=62 time=0.436 ms\n\n# node &lt;--&gt; pod fip is working\n\n# The way ip is used inside the pod is roughly as follows\n\n[root@pc-node-1 fip-vip]# k -n vpc1 exec -it vpc-1-busybox03 -- bash\n[root@vpc-1-busybox03 /]#\n[root@vpc-1-busybox03 /]#\n[root@vpc-1-busybox03 /]# ip a\n1: lo: &lt;LOOPBACK,UP,LOWER_UP&gt; mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000\nlink/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00\n    inet 127.0.0.1/8 scope host lo\n       valid_lft forever preferred_lft forever\n    inet6 ::1/128 scope host\n       valid_lft forever preferred_lft forever\n1568: eth0@if1569: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1500 qdisc noqueue state UP group default\n    link/ether 00:00:00:56:40:e5 brd ff:ff:ff:ff:ff:ff link-netnsid 0\ninet 192.168.0.5/24 brd 192.168.0.255 scope global eth0\n       valid_lft forever preferred_lft forever\n    inet 192.168.0.3/24 scope global secondary eth0  # vip here\nvalid_lft forever preferred_lft forever\n    inet6 fe80::200:ff:fe56:40e5/64 scope link\n       valid_lft forever preferred_lft forever\n\n[root@vpc-1-busybox03 /]# tcpdump -i eth0 host  192.168.0.3 -netvv\ntcpdump: listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes\n00:00:00:ed:8e:c7 &gt; 00:00:00:56:40:e5, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 62, id 44830, offset 0, flags [DF], proto ICMP (1), length 84)\n10.5.32.51 &gt; 192.168.0.3: ICMP echo request, id 177, seq 1, length 64\n00:00:00:56:40:e5 &gt; 00:00:00:ed:8e:c7, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 43962, offset 0, flags [none], proto ICMP (1), length 84)\n192.168.0.3 &gt; 10.5.32.51: ICMP echo reply, id 177, seq 1, length 64\n\n# pod internal can catch fip related icmp packets\n</code></pre>"},{"location":"en/advance/ovn-eip-fip-snat/#31-ovn-snat-corresponds-to-the-cidr-of-a-subnet","title":"3.1 ovn-snat corresponds to the CIDR of a subnet","text":"<p>This feature is designed and used in much the same way as iptables-snat</p> <pre><code># cat 03-subnet-snat.yaml\n---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: snat-for-subnet-in-vpc\nspec:\n  externalSubnet: external204\n  type: nat\n\n---\nkind: OvnSnatRule\napiVersion: kubeovn.io/v1\nmetadata:\n  name: snat-for-subnet-in-vpc\nspec:\n  ovnEip: snat-for-subnet-in-vpc\n  vpcSubnet: vpc1-subnet1 # eip corresponds to the entire network segment\n</code></pre>"},{"location":"en/advance/ovn-eip-fip-snat/#32-ovn-snat-corresponds-to-a-pod-ip","title":"3.2 ovn-snat corresponds to a pod IP","text":"<p>This feature is designed and used in much the same way as iptables-snat</p> <pre><code># cat 03-pod-snat.yaml\n---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: snat-for-pod-vpc-ip\nspec:\n  externalSubnet: external204\n  type: nat\n\n---\nkind: OvnSnatRule\napiVersion: kubeovn.io/v1\nmetadata:\n  name: snat01\nspec:\n  ovnEip: snat-for-pod-vpc-ip\n  ipName: vpc-1-busybox02.vpc1 # eip corresponds to a single pod ip\n</code></pre> <p>After the above resources are created, you can see the following resources that the snat public network feature depends on.</p> <pre><code># kubectl ko nbctl show vpc1\nrouter 87ad06fd-71d5-4ff8-a1f0-54fa3bba1a7f (vpc1)\nport vpc1-vpc1-subnet1\n        mac: \"00:00:00:ED:8E:C7\"\nnetworks: [\"192.168.0.1/24\"]\nport vpc1-external204\n        mac: \"00:00:00:EF:05:C7\"\nnetworks: [\"10.5.204.105/24\"]\ngateway chassis: [7cedd14f-265b-42e5-ac17-e03e7a1f2342 276baccb-fe9c-4476-b41d-05872a94976d fd9f140c-c45d-43db-a6c0-0d4f8ea298dd]\nnat 21d853b0-f7b4-40bd-9a53-31d2e2745739\n        external ip: \"10.5.204.115\"\nlogical ip: \"192.168.0.0/24\"\ntype: \"snat\"\nnat da77a11f-c523-439c-b1d1-72c664196a0f\n        external ip: \"10.5.204.116\"\nlogical ip: \"192.168.0.4\"\ntype: \"snat\"\n</code></pre> <pre><code>[root@pc-node-1 03-cust-vpc]# k get po -A -o wide  | grep busy\nvpc1            vpc-1-busybox01                                 1/1     Running   0                3d15h   192.168.0.2   pc-node-2   &lt;none&gt;           &lt;none&gt;\nvpc1            vpc-1-busybox02                                 1/1     Running   0                17h     192.168.0.4   pc-node-1   &lt;none&gt;           &lt;none&gt;\nvpc1            vpc-1-busybox03                                 1/1     Running   0                17h     192.168.0.5   pc-node-1   &lt;none&gt;           &lt;none&gt;\nvpc1            vpc-1-busybox04                                 1/1     Running   0                17h     192.168.0.6   pc-node-3   &lt;none&gt;           &lt;none&gt;\nvpc1            vpc-1-busybox05                                 1/1     Running   0                17h     192.168.0.7   pc-node-1   &lt;none&gt;           &lt;none&gt;\n\n# k exec -it -n vpc1            vpc-1-busybox04   bash\nkubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.\n[root@vpc-1-busybox04 /]#\n[root@vpc-1-busybox04 /]#\n[root@vpc-1-busybox04 /]# ip a\n1: lo: &lt;LOOPBACK,UP,LOWER_UP&gt; mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000\nlink/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00\n    inet 127.0.0.1/8 scope host lo\n       valid_lft forever preferred_lft forever\n    inet6 ::1/128 scope host\n       valid_lft forever preferred_lft forever\n17095: eth0@if17096: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1500 qdisc noqueue state UP group default\n    link/ether 00:00:00:76:94:55 brd ff:ff:ff:ff:ff:ff link-netnsid 0\ninet 192.168.0.6/24 brd 192.168.0.255 scope global eth0\n       valid_lft forever preferred_lft forever\n    inet6 fe80::200:ff:fe76:9455/64 scope link\n       valid_lft forever preferred_lft forever\n[root@vpc-1-busybox04 /]# ping 223.5.5.5\nPING 223.5.5.5 (223.5.5.5) 56(84) bytes of data.\n64 bytes from 223.5.5.5: icmp_seq=1 ttl=114 time=22.2 ms\n64 bytes from 223.5.5.5: icmp_seq=2 ttl=114 time=21.8 ms\n\n[root@pc-node-1 03-cust-vpc]# k exec -it -n vpc1            vpc-1-busybox02   bash\nkubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.\n[root@vpc-1-busybox02 /]#\n[root@vpc-1-busybox02 /]#\n[root@vpc-1-busybox02 /]#\n[root@vpc-1-busybox02 /]# ip a\n1: lo: &lt;LOOPBACK,UP,LOWER_UP&gt; mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000\nlink/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00\n    inet 127.0.0.1/8 scope host lo\n       valid_lft forever preferred_lft forever\n    inet6 ::1/128 scope host\n       valid_lft forever preferred_lft forever\n1566: eth0@if1567: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1500 qdisc noqueue state UP group default\n    link/ether 00:00:00:0b:e9:d0 brd ff:ff:ff:ff:ff:ff link-netnsid 0\ninet 192.168.0.4/24 brd 192.168.0.255 scope global eth0\n       valid_lft forever preferred_lft forever\n    inet6 fe80::200:ff:fe0b:e9d0/64 scope link\n       valid_lft forever preferred_lft forever\n[root@vpc-1-busybox02 /]# ping 223.5.5.5\nPING 223.5.5.5 (223.5.5.5) 56(84) bytes of data.\n64 bytes from 223.5.5.5: icmp_seq=2 ttl=114 time=22.7 ms\n64 bytes from 223.5.5.5: icmp_seq=3 ttl=114 time=22.6 ms\n64 bytes from 223.5.5.5: icmp_seq=4 ttl=114 time=22.1 ms\n^C\n--- 223.5.5.5 ping statistics ---\n4 packets transmitted, 3 received, 25% packet loss, time 3064ms\nrtt min/avg/max/mdev = 22.126/22.518/22.741/0.278 ms\n\n# the two pods can access the external network based on these two type snat resources respectively\n</code></pre>"},{"location":"en/advance/ovn-eip-fip-snat/#41-ovn-dnat-binds-a-dnat-to-a-pod","title":"4.1 ovn-dnat binds a DNAT to a pod","text":"<pre><code>kind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-static\nspec:\nexternalSubnet: underlay\ntype: nat\n---\nkind: OvnDnatRule\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-dnat\nspec:\novnEip: eip-dnat\nipName: vpc-1-busybox01.vpc1 # Note that this is the name of the pod IP CRD and it is unique\nprotocol: tcp\ninternalPort: \"22\"\nexternalPort: \"22\"\n</code></pre> <p>The configuration of OvnDnatRule is similar to that of IptablesDnatRule.</p> <pre><code># kubectl get oeip eip-dnat\nNAME       V4IP        V6IP   MAC                 TYPE   READY\neip-dnat   10.5.49.4          00:00:00:4D:CE:49   dnat   true\n\n# kubectl get odnat\nNAME                   EIP                    PROTOCOL   V4EIP        V4IP           INTERNALPORT   EXTERNALPORT   IPNAME                                READY\neip-dnat               eip-dnat               tcp        10.5.49.4    192.168.0.3    22             22             vpc-1-busybox01.vpc1                  true\n</code></pre>"},{"location":"en/advance/ovn-eip-fip-snat/#42-ovn-dnat-binds-a-dnat-to-a-vip","title":"4.2 ovn-dnat binds a DNAT to a VIP","text":"<pre><code>kind: OvnDnatRule\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-dnat\nspec:\nipType: vip  # By default, Dnat is oriented towards pod IPs. Here, it is necessary to specify that it is connected to VIP resources\novnEip: eip-dnat\nipName: test-dnat-vip\nprotocol: tcp\ninternalPort: \"22\"\nexternalPort: \"22\"\n</code></pre> <p>The configuration of OvnDnatRule is similar to that of IptablesDnatRule.</p> <pre><code># kubectl get vip test-dnat-vip\nNAME            V4IP          PV4IP   MAC                 PMAC   V6IP   PV6IP   SUBNET         READY\ntest-dnat-vip   192.168.0.4           00:00:00:D0:C0:B5                         vpc1-subnet1   true\n\n# kubectl get oeip eip-dnat\nNAME       V4IP        V6IP   MAC                 TYPE   READY\neip-dnat   10.5.49.4          00:00:00:4D:CE:49   dnat   true\n\n# kubectl get odnat eip-dnat \nNAME       EIP        PROTOCOL   V4EIP       V4IP          INTERNALPORT   EXTERNALPORT   IPNAME          READY\neip-dnat   eip-dnat   tcp        10.5.49.4   192.168.0.4   22             22             test-dnat-vip   true\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/ovn-ipsec/","title":"Encrypt inter-node communication using IPsec","text":"<p>This function is supported after v1.10.11 and v1.11.4, and the kernel version is at least 3.10.0 or above.</p>"},{"location":"en/advance/ovn-ipsec/#start-ipsec","title":"start IPsec","text":"<p>Copy the script from the Kube-OVN source code ipsec.sh, execute the command as follows, the script will call ovs-pki to generate and distribute the certificate required for encryption:</p> <pre><code>bash ipsec.sh init\n</code></pre> <p>After the execution is completed, the nodes will negotiate for a period of time to establish an IPsec tunnel. The experience value is between ten seconds and one minute.You can check the IPsec status with the following command:</p> <pre><code># bash ipsec.sh status\nPod {ovs-ovn-d7hdt} ipsec status...\nInterface name: ovn-a4718e-0 v1 (CONFIGURED)\nTunnel Type:    geneve\n  Local IP:       172.18.0.2\n  Remote IP:      172.18.0.4\n  Address Family: IPv4\n  SKB mark:       None\n  Local cert:     /etc/ipsec.d/certs/8aebd9df-46ef-47b9-85e3-73e9a765296d-cert.pem\n  Local name:     8aebd9df-46ef-47b9-85e3-73e9a765296d\n  Local key:      /etc/ipsec.d/private/8aebd9df-46ef-47b9-85e3-73e9a765296d-privkey.pem\n  Remote cert:    None\n  Remote name:    a4718e55-5b85-4f46-90e6-63527d080590\n  CA cert:        /etc/ipsec.d/cacerts/cacert.pem\n  PSK:            None\n  Custom Options: {}\nOfport:         2\nCFM state:      Disabled\nKernel policies installed:\n  src 172.18.0.2/32 dst 172.18.0.4/32 proto udp sport 6081\nsrc 172.18.0.2/32 dst 172.18.0.4/32 proto udp sport 6081\nsrc 172.18.0.2/32 dst 172.18.0.4/32 proto udp dport 6081\nsrc 172.18.0.2/32 dst 172.18.0.4/32 proto udp dport 6081\nKernel security associations installed:\n  sel src 172.18.0.2/32 dst 172.18.0.4/32 proto udp sport 6081\nsel src 172.18.0.4/32 dst 172.18.0.2/32 proto udp dport 6081\nsel src 172.18.0.2/32 dst 172.18.0.4/32 proto udp dport 6081\nsel src 172.18.0.4/32 dst 172.18.0.2/32 proto udp sport 6081\nIPsec connections that are active:\n\n Pod {ovs-ovn-fvbbj} ipsec status...\nInterface name: ovn-8aebd9-0 v1 (CONFIGURED)\nTunnel Type:    geneve\n  Local IP:       172.18.0.4\n  Remote IP:      172.18.0.2\n  Address Family: IPv4\n  SKB mark:       None\n  Local cert:     /etc/ipsec.d/certs/a4718e55-5b85-4f46-90e6-63527d080590-cert.pem\n  Local name:     a4718e55-5b85-4f46-90e6-63527d080590\n  Local key:      /etc/ipsec.d/private/a4718e55-5b85-4f46-90e6-63527d080590-privkey.pem\n  Remote cert:    None\n  Remote name:    8aebd9df-46ef-47b9-85e3-73e9a765296d\n  CA cert:        /etc/ipsec.d/cacerts/cacert.pem\n  PSK:            None\n  Custom Options: {}\nOfport:         1\nCFM state:      Disabled\nKernel policies installed:\n  src 172.18.0.4/32 dst 172.18.0.2/32 proto udp dport 6081\nsrc 172.18.0.4/32 dst 172.18.0.2/32 proto udp dport 6081\nsrc 172.18.0.4/32 dst 172.18.0.2/32 proto udp sport 6081\nsrc 172.18.0.4/32 dst 172.18.0.2/32 proto udp sport 6081\nKernel security associations installed:\n  sel src 172.18.0.4/32 dst 172.18.0.2/32 proto udp dport 6081\nsel src 172.18.0.2/32 dst 172.18.0.4/32 proto udp sport 6081\nsel src 172.18.0.4/32 dst 172.18.0.2/32 proto udp sport 6081\nsel src 172.18.0.2/32 dst 172.18.0.4/32 proto udp dport 6081\nIPsec connections that are active:\n</code></pre> <p>After the establishment is complete, you can capture packets and observe that the packets have been encrypted:</p> <pre><code># tcpdump -i eth0 -nel esp\n10:01:40.349896 IP kube-ovn-worker &gt; kube-ovn-control-plane.kind: ESP(spi=0xcc91322a,seq=0x13d0), length 156\n10:01:40.350015 IP kube-ovn-control-plane.kind &gt; kube-ovn-worker: ESP(spi=0xc8df4221,seq=0x1d37), length 156\n</code></pre> <p>After executing the script, you can turn off IPsec by executing the command:</p> <pre><code># bash ipsec.sh stop\n</code></pre> <p>Or execute the command to open it again:</p> <pre><code># bash ipsec.sh start\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/ovn-l3-ha-based-ecmp-with-bfd/","title":"Support OVN SNAT L3 HA Based ECMP and BFD Static Route","text":"<p>Custom vpc based on ovn snat after ecmp based static route hash to multiple gw node ovnext0 NICs out of the public network</p> <ul> <li>Supports bfd-based high availability</li> <li>Only supports hash load balancing</li> </ul> <pre><code>graph LR\n\npod--&gt;vpc-subnet--&gt;vpc--&gt;snat--&gt;ecmp--&gt;external-subnet--&gt;gw-node1-ovnext0--&gt; node1-external-switch\nexternal-subnet--&gt;gw-node2-ovnext0--&gt; node2-external-switch\nexternal-subnet--&gt;gw-node3-ovnext0--&gt; node3-external-switch</code></pre> <p>This functions basically the same as ovn-eip-fip-snat.md .</p> <p>As for the different parts, which will be specified in the following sections, mainly including the creation of ovn-eip of lsp type and the automatic maintenance of bfd as well as ecmp static routes based on vpc enable_bfd.</p>"},{"location":"en/advance/ovn-l3-ha-based-ecmp-with-bfd/#1-deployment","title":"1. Deployment","text":""},{"location":"en/advance/ovn-l3-ha-based-ecmp-with-bfd/#11-create-the-underlay-public-network","title":"1.1 Create the underlay public network","text":""},{"location":"en/advance/ovn-l3-ha-based-ecmp-with-bfd/#12-default-vpc-enable-eip_snat","title":"1.2 Default vpc enable eip_snat","text":""},{"location":"en/advance/ovn-l3-ha-based-ecmp-with-bfd/#13-custom-vpc-enable-eip-snat-fip-function","title":"1.3 Custom vpc enable eip snat fip function","text":"<p>The above section is exactly the same with ovn-eip-fip-snat.md.</p> <p>After these functions are verified, the vpc can be switched directly to the ecmp-based bfd static route based on the following way, or of course, switched directly back.</p> <p>Before customizing vpc to use this feature, you need to provide some gateway nodes, at least 2. Note that the name of the current implementation of ovn-eip must be consistent with the gateway node name, no automated maintenance is currently done for this resource.</p> <pre><code># cat gw-node-eip.yaml\n---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\nname: pc-node-1\nspec:\nexternalSubnet: external204\ntype: lsp\n\n---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\nname: pc-node-2\nspec:\nexternalSubnet: external204\ntype: lsp\n\n---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\nname: pc-node-3\nspec:\nexternalSubnet: external204\ntype: lsp\n</code></pre> <p>Since this scenario is currently designed for vpc ecmp out of the public network, the gateway node above will not trigger the creation of a gateway NIC when there is no vpc enabled bfd, i.e. when there is no ovn eip (lrp) with enable bfd labeled, and will not be able to successfully start listening to the bfd session on the other side.</p>"},{"location":"en/advance/ovn-l3-ha-based-ecmp-with-bfd/#2-custom-vpc-enable-ecmp-bfd-l3-ha-public-network-function","title":"2. Custom vpc enable ecmp bfd L3 HA public network function","text":"<pre><code># cat 01-vpc-ecmp-enable-external-bfd.yml\nkind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\n  name: vpc1\nspec:\n  namespaces:\n  - vpc1\n  enableExternal: true\nenableBfd: true # bfd switch can be switched at will\n#enableBfd: false \n\n\n# cat 02-subnet.yml\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n  name: vpc1-subnet1\nspec:\n  cidrBlock: 192.168.0.0/24\n  default: false\ndisableGatewayCheck: false\ndisableInterConnection: true\nenableEcmp: true  # enable ecmp\ngatewayNode: \"\"\ngatewayType: distributed\n  #gatewayType: centralized\nnatOutgoing: false\nprivate: false\nprotocol: IPv4\n  provider: ovn\n  vpc: vpc1\n  namespaces:\n  - vpc1\n</code></pre> <p>note:</p> <ol> <li>Customize ecmp under vpc to use only static ecmp bfd routes. vpc enableBfd and subnet enableEcmp will only take effect if they are enabled at the same time, before static ecmp bfd routes are automatically managed.</li> <li>If the above configuration is turned off, it will automatically switch back to the regular default static route.</li> <li>This feature is not available for the default vpc, only custom vpc is supported, the default vpc has more complex policy routing.</li> <li>The enableEcmp of the subnet of the custom vpc uses only static routes, the gateway type gatewayType has no effect.</li> <li>When EnableExternal is turned off in vpc, the external network cannot be passed inside vpc.</li> <li>When EnableExternal is enabled on vpc, when EnableBfd is turned off, it will be based on the normal default route to the external network and will not have high availability.</li> </ol> <pre><code># After the above template is applied the ovn logic layer should see the following resources\n# k get vpc\nNAME          ENABLEEXTERNAL   ENABLEBFD   STANDBY   SUBNETS                                NAMESPACES\novn-cluster   true                         true      [\"external204\",\"join\",\"ovn-default\"]\nvpc1          true             true        true      [\"vpc1-subnet1\"]                       [\"vpc1\"]\n\n# Default vpc does not support ENABLEBFD\n# Custom vpc is supported and enabled\n\n# 1. bfd table created\n# k ko nbctl list bfd\n_uuid               : be7df545-2c4c-4751-878f-b3507987f050\ndetect_mult         : 3\ndst_ip              : \"10.5.204.121\"\nexternal_ids        : {}\nlogical_port        : vpc1-external204\nmin_rx              : 100\nmin_tx              : 100\noptions             : {}\nstatus              : up\n\n_uuid               : 684c4489-5b59-4693-8d8c-3beab93f8093\ndetect_mult         : 3\ndst_ip              : \"10.5.204.109\"\nexternal_ids        : {}\nlogical_port        : vpc1-external204\nmin_rx              : 100\nmin_tx              : 100\noptions             : {}\nstatus              : up\n\n_uuid               : f0f62077-2ae9-4e79-b4f8-a446ec6e784c\ndetect_mult         : 3\ndst_ip              : \"10.5.204.108\"\nexternal_ids        : {}\nlogical_port        : vpc1-external204\nmin_rx              : 100\nmin_tx              : 100\noptions             : {}\nstatus              : up\n\n### Note that all statuses should normally be up\n\n# 2. bfd ecmp static routes table created\n# k ko nbctl lr-route-list vpc1\nIPv4 Routes\nRoute Table &lt;main&gt;:\n           192.168.0.0/24              10.5.204.108 src-ip ecmp ecmp-symmetric-reply bfd\n           192.168.0.0/24              10.5.204.109 src-ip ecmp ecmp-symmetric-reply bfd\n           192.168.0.0/24              10.5.204.121 src-ip ecmp ecmp-symmetric-reply bfd\n\n# 3. Static Route Details\n# k ko nbctl find Logical_Router_Static_Route  policy=src-ip options=ecmp_symmetric_reply=\"true\"\n_uuid               : 3aacb384-d5ee-4b14-aebf-59e8c11717ba\nbfd                 : 684c4489-5b59-4693-8d8c-3beab93f8093\nexternal_ids        : {}\nip_prefix           : \"192.168.0.0/24\"\nnexthop             : \"10.5.204.109\"\noptions             : {ecmp_symmetric_reply=\"true\"}\noutput_port         : []\npolicy              : src-ip\nroute_table         : \"\"\n\n_uuid               : 18bcc585-bc05-430b-925b-ef673c8e1aef\nbfd                 : f0f62077-2ae9-4e79-b4f8-a446ec6e784c\nexternal_ids        : {}\nip_prefix           : \"192.168.0.0/24\"\nnexthop             : \"10.5.204.108\"\noptions             : {ecmp_symmetric_reply=\"true\"}\noutput_port         : []\npolicy              : src-ip\nroute_table         : \"\"\n\n_uuid               : 7d0a4e6b-cde0-4110-8176-fbaf19738498\nbfd                 : be7df545-2c4c-4751-878f-b3507987f050\nexternal_ids        : {}\nip_prefix           : \"192.168.0.0/24\"\nnexthop             : \"10.5.204.121\"\noptions             : {ecmp_symmetric_reply=\"true\"}\noutput_port         : []\npolicy              : src-ip\nroute_table         : \"\"\n</code></pre> <pre><code># Also, the following resources should be available at all gateway nodes\n\n[root@pc-node-1 ~]# ip netns exec ovnext bash ip a\n/usr/sbin/ip: /usr/sbin/ip: cannot execute binary file\n[root@pc-node-1 ~]#\n[root@pc-node-1 ~]# ip netns exec ovnext ip a\n1: lo: &lt;LOOPBACK,UP,LOWER_UP&gt; mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000\nlink/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00\n    inet 127.0.0.1/8 scope host lo\n       valid_lft forever preferred_lft forever\n    inet6 ::1/128 scope host\n       valid_lft forever preferred_lft forever\n1541: ovnext0: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1400 qdisc noqueue state UNKNOWN group default qlen 1000\nlink/ether 00:00:00:ab:bd:87 brd ff:ff:ff:ff:ff:ff\n    inet 10.5.204.108/24 brd 10.5.204.255 scope global ovnext0\n       valid_lft forever preferred_lft forever\n    inet6 fe80::200:ff:feab:bd87/64 scope link\n       valid_lft forever preferred_lft forever\n[root@pc-node-1 ~]#\n[root@pc-node-1 ~]# ip netns exec ovnext route -n\nKernel IP routing table\nDestination     Gateway         Genmask         Flags Metric Ref    Use Iface\n0.0.0.0         10.5.204.254    0.0.0.0         UG    0      0        0 ovnext0\n10.5.204.0      0.0.0.0         255.255.255.0   U     0      0        0 ovnext0\n\n\n[root@pc-node-1 ~]# ip netns exec ovnext bfdd-control status\nThere are 1 sessions:\nSession 1\nid=1 local=10.5.204.108 (p) remote=10.5.204.122 state=Up\n\n## This is the other end of the lrp bfd session and one of the next hops of the lrp ecmp\n\n\n[root@pc-node-1 ~]# ip netns exec ovnext ping -c1 223.5.5.5\nPING 223.5.5.5 (223.5.5.5) 56(84) bytes of data.\n64 bytes from 223.5.5.5: icmp_seq=1 ttl=115 time=21.6 ms\n\n# No problem to the public network\n</code></pre> <p>catch outgoing packets within the ovnext ns of a gateway node</p> <pre><code># tcpdump -i ovnext0 host 223.5.5.5 -netvv\ndropped privs to tcpdump\ntcpdump: listening on ovnext0, link-type EN10MB (Ethernet), capture size 262144 bytes\n^C\n0 packets captured\n0 packets received by filter\n0 packets dropped by kernel\n[root@pc-node-1 ~]# exit\n[root@pc-node-1 ~]# ssh pc-node-2\nLast login: Thu Feb 23 09:21:08 2023 from 10.5.32.51\n[root@pc-node-2 ~]# ip netns exec ovnext bash\n[root@pc-node-2 ~]# tcpdump -i ovnext0 host 223.5.5.5 -netvv\ndropped privs to tcpdump\ntcpdump: listening on ovnext0, link-type EN10MB (Ethernet), capture size 262144 bytes\n^C\n0 packets captured\n0 packets received by filter\n0 packets dropped by kernel\n[root@pc-node-2 ~]# exit\n[root@pc-node-2 ~]# logout\nConnection to pc-node-2 closed.\n[root@pc-node-1 ~]# ssh pc-node-3\nLast login: Thu Feb 23 08:32:41 2023 from 10.5.32.51\n[root@pc-node-3 ~]#  ip netns exec ovnext bash\n[root@pc-node-3 ~]# tcpdump -i ovnext0 host 223.5.5.5 -netvv\ndropped privs to tcpdump\ntcpdump: listening on ovnext0, link-type EN10MB (Ethernet), capture size 262144 bytes\n00:00:00:2d:f8:ce &gt; 00:00:00:fd:b2:a4, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 57978, offset 0, flags [DF], proto ICMP (1), length 84)\n10.5.204.102 &gt; 223.5.5.5: ICMP echo request, id 22, seq 71, length 64\n00:00:00:fd:b2:a4 &gt; dc:ef:80:5a:44:1a, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 62, id 57978, offset 0, flags [DF], proto ICMP (1), length 84)\n10.5.204.102 &gt; 223.5.5.5: ICMP echo request, id 22, seq 71, length 64\n^C\n2 packets captured\n2 packets received by filter\n0 packets dropped by kernel\n[root@pc-node-3 ~]#\n</code></pre>"},{"location":"en/advance/ovn-l3-ha-based-ecmp-with-bfd/#3-turn-off-bfd-mode","title":"3. Turn off bfd mode","text":"<p>In some scenarios, you may want to use a (centralized) single gateway directly out of the public network, which is the same as the default vpc enable_eip_snat usage pattern</p> <pre><code># cat 01-vpc-ecmp-enable-external-bfd.yml\nkind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\n  name: vpc2\nspec:\n  namespaces:\n  - vpc2\n  enableExternal: true\n#enableBfd: true\nenableBfd: false\n\n## set it false add apply\n\n# k ko nbctl lr-route-list vpc2\nIPv4 Routes\nRoute Table &lt;main&gt;:\n                0.0.0.0/0              10.5.204.254 dst-ip\n\n# After application the route will switch back to the normal default static route\n# nbctl list bfd, the bfd session associated with lrp has been removed\n# And the opposite side of the bfd session in ovnext ns is automatically removed\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/ovn-remote-port-mirroring/","title":"OVN Remote Port Mirroring","text":"<p>This feature provides ability to mirror the traffic of the specified Pod and direction, and to send the mirrored traffic to a remote destination.</p> <p>This feature requires Kube-OVN version not lower than v1.12.</p>"},{"location":"en/advance/ovn-remote-port-mirroring/#install-multus-cni","title":"Install Multus-CNI","text":"<p>Install Multus-CNI by referring the Multus-CNI Document.</p>"},{"location":"en/advance/ovn-remote-port-mirroring/#create-networkattachmentdefinition","title":"Create NetworkAttachmentDefinition","text":"<p>Create the following NetworkAttachmentDefinition:</p> <pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: attachnet\nnamespace: default\nspec:\nconfig: |\n{\n\"cniVersion\": \"0.3.1\",\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"attachnet.default.ovn\"\n}\n</code></pre> <p>Format of the <code>provider</code> field is <code>&lt;NAME&gt;.&lt;NAMESPACE&gt;.ovn</code>.</p>"},{"location":"en/advance/ovn-remote-port-mirroring/#create-underlay-network","title":"Create Underlay Network","text":"<p>The mirrored traffic is encapsulated before transmition, so MTU of the network used to transmit the traffic should be greater than the mirrored LSP/Pod. Here we are using an underlay network.</p> <p>Create the following underlay network:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: ProviderNetwork\nmetadata:\nname: net1\nspec:\ndefaultInterface: eth1\n---\napiVersion: kubeovn.io/v1\nkind: Vlan\nmetadata:\nname: vlan1\nspec:\nid: 0\nprovider: net1\n---\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: subnet1\nspec:\nprotocol: IPv4\ncidrBlock: 172.19.0.0/16\nexcludeIps:\n- 172.19.0.2..172.19.0.20\ngateway: 172.19.0.1\nvlan: vlan1\nprovider: attachnet.default.ovn\n</code></pre> <p>The subnet's <code>provider</code> MUST be the same as the <code>provider</code> of the NetworkAttachmentDefinition created above.</p>"},{"location":"en/advance/ovn-remote-port-mirroring/#create-receiving-pod","title":"Create Receiving Pod","text":"<p>Create the following Pod\uff1a</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: pod1\nannotations:\nk8s.v1.cni.cncf.io/networks: default/attachnet\nspec:\ncontainers:\n- name: bash\nimage: docker.io/kubeovn/kube-ovn:v1.13.0\nargs:\n- bash\n- -c\n- sleep infinity\nsecurityContext:\nprivileged: true\n</code></pre> <p>After the Pod has been created, checkout the IP addresses:</p> <pre><code>$ kubectl get ips | grep pod1\npod1.default                        10.16.0.12   00:00:00:FF:34:24  kube-ovn-worker  ovn-default\npod1.default.attachnet.default.ovn  172.19.0.21  00:00:00:A0:30:68  kube-ovn-worker  subnet1\n</code></pre> <p>The IP address <code>172.19.0.21</code> will be used later.</p>"},{"location":"en/advance/ovn-remote-port-mirroring/#create-ovn-remote-port-mirroring","title":"Create OVN Remote Port Mirroring","text":"<p>Create the following OVN remote port mirroring\uff1a</p> <pre><code>kubectl ko nbctl mirror-add mirror1 gre 99 from-lport 172.19.0.21\nkubectl ko nbctl lsp-attach-mirror coredns-787d4945fb-gpnkb.kube-system mirror1\n</code></pre> <p><code>coredns-787d4945fb-gpnkb.kube-system</code> is the OVN LSP name with a format <code>&lt;POD_NAME&gt;.&lt;POD_NAMESPACE&gt;</code>.</p> <p>Here is the OVN command usage:</p> <pre><code>ovn-nbctl mirror-add &lt;NAME&gt; &lt;TYPE&gt; &lt;INDEX&gt; &lt;FILTER&gt; &lt;IP&gt;\n\nNAME   - add a mirror with given name\nTYPE   - specify TYPE 'gre' or 'erspan'\nINDEX  - specify the tunnel INDEX value\n         (indicates key if GRE, erpsan_idx if ERSPAN)\nFILTER - specify FILTER for mirroring selection\n         ('to-lport' / 'from-lport')\nIP     - specify Sink / Destination i.e. Remote IP\n\novn-nbctl mirror-del [NAME]         remove mirrors\novn-nbctl mirror-list               print mirrors\n\novn-nbctl lsp-attach-mirror PORT MIRROR   attach source PORT to MIRROR\novn-nbctl lsp-detach-mirror PORT MIRROR   detach source PORT from MIRROR\n</code></pre>"},{"location":"en/advance/ovn-remote-port-mirroring/#configure-receiving-pod","title":"Configure Receiving Pod","text":"<p>Execute the following commands in the Pod:</p> <pre><code>root@pod1:/kube-ovn# ip link add mirror1 type gretap local 172.19.0.21 key 99 dev net1\nroot@pod1:/kube-ovn# ip link set mirror1 up\n</code></pre> <p>Now you can capture the mirrored packets:</p> <pre><code>root@pod1:/kube-ovn# tcpdump -i mirror1 -nnve\ntcpdump: listening on mirror1, link-type EN10MB (Ethernet), snapshot length 262144 bytes\n05:13:30.328808 00:00:00:a3:f5:e2 &gt; 00:00:00:97:0f:6e, ethertype ARP (0x0806), length 42: Ethernet (len 6), IPv4 (len 4), Request who-has 10.16.0.7 tell 10.16.0.4, length 28\n05:13:30.559167 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 212: (tos 0x0, ttl 64, id 57364, offset 0, flags [DF], proto UDP (17), length 198)\n10.16.0.4.53 &gt; 10.16.0.6.50472: 34511 NXDomain*- 0/1/1 (170)\n05:13:30.559343 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 212: (tos 0x0, ttl 64, id 57365, offset 0, flags [DF], proto UDP (17), length 198)\n10.16.0.4.53 &gt; 10.16.0.6.45177: 1659 NXDomain*- 0/1/1 (170)\n05:13:30.560625 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 200: (tos 0x0, ttl 64, id 57367, offset 0, flags [DF], proto UDP (17), length 186)\n10.16.0.4.53 &gt; 10.16.0.6.43848: 2636*- 0/1/1 (158)\n05:13:30.562774 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 191: (tos 0x0, ttl 64, id 57368, offset 0, flags [DF], proto UDP (17), length 177)\n10.16.0.4.53 &gt; 10.16.0.6.37755: 48737 NXDomain*- 0/1/1 (149)\n05:13:30.563523 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 187: (tos 0x0, ttl 64, id 57369, offset 0, flags [DF], proto UDP (17), length 173)\n10.16.0.4.53 &gt; 10.16.0.6.53887: 45519 NXDomain*- 0/1/1 (145)\n05:13:30.564940 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 201: (tos 0x0, ttl 64, id 57370, offset 0, flags [DF], proto UDP (17), length 187)\n10.16.0.4.53 &gt; 10.16.0.6.40846: 25745 NXDomain*- 0/1/1 (159)\n05:13:30.565140 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 201: (tos 0x0, ttl 64, id 57371, offset 0, flags [DF], proto UDP (17), length 187)\n10.16.0.4.53 &gt; 10.16.0.6.45214: 61875 NXDomain*- 0/1/1 (159)\n05:13:30.566023 00:00:00:a3:f5:e2 &gt; 00:00:00:55:e4:4e, ethertype IPv4 (0x0800), length 80: (tos 0x0, ttl 64, id 45937, offset 0, flags [DF], proto UDP (17), length 66)\n10.16.0.4.44116 &gt; 172.18.0.1.53: 16025+ [1au] AAAA? alauda.cn. (38)\n</code></pre>"},{"location":"en/advance/ovn-remote-port-mirroring/#notice","title":"Notice","text":"<ol> <li>If you are using ERSPAN as the encapsulation protocol, the Linux kernel version of the OVN nodes and remote devices must not be lower than 4.14. If you are using ERSPAN as the encapsulation protocol and using IPv6 as the transport network, the Linux kernel version must not be lower than 4.16.</li> <li>The transmission of mirrored traffic is unidirectional, so you only need to ensure that the OVN node can access the remote device.</li> </ol> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/performance-tuning/","title":"Performance Tuning","text":"<p>To keep the installation simple and feature-complete, the default installation script for Kube-OVN does not have performance-specific optimizations. If the applications are sensitive to latency and throughput, administrators can use this document to make specific performance optimizations.</p> <p>The community will continue to iterate on the performance. Some general performance optimizations have been integrated into the latest version, so it is recommended to use the latest version to get better default performance.</p> <p>For more on the process and methodology of performance optimization, please watch the video Kube-OVN \u5bb9\u5668\u6027\u80fd\u4f18\u5316\u4e4b\u65c5\u3002</p>"},{"location":"en/advance/performance-tuning/#benchmarking","title":"Benchmarking","text":"<p>Because the hardware and software environments vary greatly, the performance test data provided here can only be used as a reference, and the actual test results may differ significantly from the results in this document. It is recommended to compare the performance test results before and after optimization, and the performance comparison between the host network and the container network.</p>"},{"location":"en/advance/performance-tuning/#overlay-performance-comparison-before-and-after-optimization","title":"Overlay Performance Comparison before and after Optimization","text":"<p>Environment:</p> <ul> <li>Kubernetes: 1.22.0</li> <li>OS: CentOS 7</li> <li>Kube-OVN: 1.8.0 Overlay Mode</li> <li>CPU: Intel(R) Xeon(R) E-2278G</li> <li>Network: 2*10Gbps, xmit_hash_policy=layer3+4</li> </ul> <p>We use <code>qperf -t 60 &lt;server ip&gt; -ub -oo msg_size:1 -vu tcp_lat tcp_bw udp_lat udp_bw</code> to test bandwidth and latency of tcp/udp in 1-byte packets and the host network, respectively.</p> Type tcp_lat (us) udp_lat (us) tcp_bw (Mb/s) udp_bw(Mb/s) Kube-OVN Default 25.7 22.9 27.1 1.59 Kube-OVN Optimized 13.9 12.9 27.6 5.57 HOST Network 13.1 12.4 28.2 6.02"},{"location":"en/advance/performance-tuning/#overlay-underlay-and-calico-comparison","title":"Overlay\uff0c Underlay and Calico Comparison","text":"<p>Next, we compare the overlay and underlay performance of the optimized Kube-OVN at different packet sizes with Calico's <code>IPIP Always</code>, <code>IPIP never</code> and the host network.</p> <p>Environment:</p> <ul> <li>Kubernetes: 1.22.0</li> <li>OS: CentOS 7</li> <li>Kube-OVN: 1.8.0</li> <li>CPU: AMD EPYC 7402P 24-Core Processor</li> <li>Network: Intel Corporation Ethernet Controller XXV710 for 25GbE SFP28</li> </ul> <p><code>qperf -t 60 &lt;server ip&gt; -ub -oo msg_size:1 -vu tcp_lat tcp_bw udp_lat udp_bw</code></p> Type tcp_lat (us) udp_lat (us) tcp_bw (Mb/s) udp_bw(Mb/s) Kube-OVN Overlay 15.2 14.6 23.6 2.65 Kube-OVN Underlay 14.3 13.8 24.2 3.46 Calico IPIP 21.4 20.2 23.6 1.18 Calico NoEncap 19.3 16.9 23.6 1.76 HOST Network 16.6 15.4 24.8 2.64 <p><code>qperf -t 60 &lt;server ip&gt; -ub -oo msg_size:1K -vu tcp_lat tcp_bw udp_lat udp_bw</code></p> Type tcp_lat (us) udp_lat (us) tcp_bw (Gb/s) udp_bw(Gb/s) Kube-OVN Overlay 16.5 15.8 10.2 2.77 Kube-OVN Underlay 15.9 14.5 9.6 3.22 Calico IPIP 22.5 21.5 1.45 1.14 Calico NoEncap 19.4 18.3 3.76 1.63 HOST Network 18.1 16.6 9.32 2.66 <p><code>qperf -t 60 &lt;server ip&gt; -ub -oo msg_size:4K -vu tcp_lat tcp_bw udp_lat udp_bw</code></p> Type tcp_lat (us) udp_lat (us) tcp_bw (Gb/s) udp_bw(Gb/s) Kube-OVN Overlay 34.7 41.6 16.0 9.23 Kube-OVN Underlay 32.6 44 15.1 6.71 Calico IPIP 44.8 52.9 2.94 3.26 Calico NoEncap 40 49.6 6.56 4.19 HOST Network 35.9 45.9 14.6 5.59 <p>In some cases the container network outperforms the host network, this is because the container network path is optimized to completely bypass netfilter. Due to the existence of <code>kube-proxy</code>, all packets in host network have to go through netfilter, which will lead to more CPU consumption, so that container network in some environments has better performance.</p>"},{"location":"en/advance/performance-tuning/#dataplane-performance-optimization-methods","title":"Dataplane performance optimization methods","text":"<p>The optimization methods described here are related to the hardware and software environment and the desired functionality, so please carefully understand the prerequisites for optimization before attempting it.</p>"},{"location":"en/advance/performance-tuning/#cpu-performance-mode-tuning","title":"CPU Performance Mode Tuning","text":"<p>In some environments the CPU is running in power saving mode, performance in this mode will be unstable and latency will increase significantly, it is recommended to use the CPU's performance mode for more stable performance.</p> <pre><code>cpupower frequency-set -g performance\n</code></pre>"},{"location":"en/advance/performance-tuning/#nic-hardware-queue-adjustment","title":"NIC Hardware Queue Adjustment","text":"<p>In the case of increased traffic, a small buffer queue may lead to significant performance degradation due to a high packet loss rate and needs to be tuned.</p> <p>Check the current NIC queue length:</p> <pre><code># ethtool -g eno1\nRing parameters for eno1:\n Pre-set maximums:\n RX:             4096\nRX Mini:        0\nRX Jumbo:       0\nTX:             4096\nCurrent hardware settings:\n RX:             255\nRX Mini:        0\nRX Jumbo:       0\nTX:             255\n</code></pre> <p>Increase the queue length to the maximum:</p> <pre><code>ethtool -G eno1 rx 4096\nethtool -G eno1 tx 4096\n</code></pre>"},{"location":"en/advance/performance-tuning/#optimize-with-tuned","title":"Optimize with tuned","text":"<p>tuned can use a series of preconfigured profile files to perform system optimizations for a specific scenario.</p> <p>For latency-first scenarios:</p> <pre><code>tuned-adm profile network-latency\n</code></pre> <p>For throughput-first scenarios:</p> <pre><code>tuned-adm profile network-throughput\n</code></pre>"},{"location":"en/advance/performance-tuning/#interrupt-binding","title":"Interrupt Binding","text":"<p>We recommend disabling <code>irqbalance</code> and binding NIC interrupts to specific CPUs to avoid performance fluctuations caused by switching between multiple CPUs.</p>"},{"location":"en/advance/performance-tuning/#disable-ovn-lb","title":"Disable OVN LB","text":"<p>The L2 LB implementation of OVN requires calling the kernel's <code>conntrack</code> module and recirculate, resulting in a significant CPU overhead, which is tested to be around 20%. For Overlay networks you can use <code>kube-proxy</code> to complete the service forwarding function for better Pod-to-Pod performance. This can be turned off in <code>kube-ovn-controller</code> args:</p> <pre><code>command:\n- /kube-ovn/start-controller.sh\nargs:\n...\n- --enable-lb=false\n...\n</code></pre> <p>In Underlay mode <code>kube-proxy</code> cannot use iptables or ipvs to control container network traffic, if you want to disable the LB function, you need to confirm whether you do not need the Service function.</p>"},{"location":"en/advance/performance-tuning/#fastpath-kernel-module","title":"FastPath Kernel Module","text":"<p>Since the container network and the host network are on different network ns, the packets will pass through the netfilter module several times when they are transmitted across the host, which results in a CPU overhead of nearly 20%. The <code>FastPath</code> module can reduce CPU overhead by bypassing netfilter, since in most cases applications within a container network do not need to use the functionality of the netfilter module.</p> <p>If you need to use the functions provided by netfilter such as iptables, ipvs, nftables, etc. in the container network, this module will disable the related functions.</p> <p>Since kernel modules are kernel version dependent, it is not possible to provide a single kernel module artifact that adapts to all kernels. We pre-compiled the <code>FastPath</code> module for part of the kernels, which can be accessed by tunning-package.</p> <p>You can also compile it manually, see Compiling FastPath Module</p> <p>After obtaining the kernel module, you can load the <code>FastPath</code> module on each node using <code>insmod kube_ovn_fastpath.ko</code> and verify that the module was loaded successfully using <code>dmesg</code>:</p> <pre><code># dmesg\n...\n[619631.323788] init_module,kube_ovn_fastpath_local_out\n[619631.323798] init_module,kube_ovn_fastpath_post_routing\n[619631.323800] init_module,kube_ovn_fastpath_pre_routing\n[619631.323801] init_module,kube_ovn_fastpath_local_in\n...\n</code></pre>"},{"location":"en/advance/performance-tuning/#ovs-kernel-module-optimization","title":"OVS Kernel Module Optimization","text":"<p>OVS flow processing including hashing, matching, etc. consumes about 10% of the CPU resources. Some instruction sets on modern x86 CPUs such as <code>popcnt</code> and <code>sse4.2</code> can speed up the computation process, but the kernel is not compiled with these options enabled. It has been tested that the CPU consumption of flow-related operations is reduced to about 5% when the corresponding instruction set optimizations are enabled.</p> <p>Similar to the compilation of the <code>FastPath</code> module, it is not possible to provide a single kernel module artifact for all kernels. Users need to compile manually or go to tunning-package to see if a compiled package is available for download.</p> <p>Before using this kernel module, please check if the CPU supports the following instruction set:</p> <pre><code>cat /proc/cpuinfo  | grep popcnt\ncat /proc/cpuinfo  | grep sse4_2\n</code></pre>"},{"location":"en/advance/performance-tuning/#compile-and-install-in-centos","title":"Compile and Install in CentOS","text":"<p>Install the relevant compilation dependencies and kernel headers:</p> <pre><code>yum install -y gcc kernel-devel-$(uname -r) python3 autoconf automake libtool rpm-build openssl-devel\n</code></pre> <p>Compile the OVS kernel module and generate the corresponding RPM:</p> <pre><code>git clone -b branch-2.17 --depth=1 https://github.com/openvswitch/ovs.git\ncd ovs\ncurl -s  https://github.com/kubeovn/ovs/commit/2d2c83c26d4217446918f39d5cd5838e9ac27b32.patch |  git apply\n./boot.sh\n./configure --with-linux=/lib/modules/$(uname -r)/build CFLAGS=\"-g -O2 -mpopcnt -msse4.2\"\nmake rpm-fedora-kmod\ncd rpm/rpmbuild/RPMS/x86_64/\n</code></pre> <p>Copy the RPM to each node and install:</p> <pre><code>rpm -i openvswitch-kmod-2.15.2-1.el7.x86_64.rpm\n</code></pre> <p>If you have previously started Kube-OVN and the older version of the OVS module has been loaded into the kernel. It is recommended to reboot the machine to reload the new version of the kernel module.</p>"},{"location":"en/advance/performance-tuning/#compile-and-install-in-ubuntu","title":"Compile and Install in Ubuntu","text":"<p>Install the relevant compilation dependencies and kernel headers:</p> <pre><code>apt install -y autoconf automake libtool gcc build-essential libssl-dev\n</code></pre> <p>Compile the OVS kernel module and install:</p> <pre><code>apt install -y autoconf automake libtool gcc build-essential libssl-dev\n\ngit clone -b branch-2.17 --depth=1 https://github.com/openvswitch/ovs.git\ncd ovs\ncurl -s  https://github.com/kubeovn/ovs/commit/2d2c83c26d4217446918f39d5cd5838e9ac27b32.patch |  git apply\n./boot.sh\n./configure --prefix=/usr/ --localstatedir=/var --enable-ssl --with-linux=/lib/modules/$(uname -r)/build\nmake -j `nproc`\nmake install\nmake modules_install\n\ncat &gt; /etc/depmod.d/openvswitch.conf &lt;&lt; EOF\noverride openvswitch * extra\noverride vport-* * extra\nEOF\n\ndepmod -a\ncp debian/openvswitch-switch.init /etc/init.d/openvswitch-switch\n/etc/init.d/openvswitch-switch force-reload-kmod\n</code></pre> <p>If you have previously started Kube-OVN and the older version of the OVS module has been loaded into the kernel. It is recommended to reboot the machine to reload the new version of the kernel module.</p>"},{"location":"en/advance/performance-tuning/#using-stt-type-tunnel","title":"Using STT Type Tunnel","text":"<p>Common tunnel encapsulation protocols such as Geneve and Vxlan use the UDP protocol to encapsulate packets and are well supported in the kernel. However, when TCP packets are encapsulated using UDP, the optimization and offload features of modern operating systems and network cards for the TCP protocol do not work well, resulting in a significant drop in TCP throughput. In some virtualization scenarios, due to CPU limitations, TCP packet throughput may even be a tenth of that of the host network.</p> <p>STT provides an innovative tunneling protocol that uses TCP formatted header for encapsulation. This encapsulation only emulates the TCP protocol header format without actually establishing a TCP connection, but can take full advantage of the TCP optimization capabilities of modern operating systems and network cards. In our tests TCP packet throughput can be improved several times, reaching performance levels close to those of the host network.</p> <p>The STT tunnel is not pre-installed in the kernel and needs to be installed by compiling the OVS kernel module, which can be found in the previous section.</p> <p>Enable STT tunnel:</p> <pre><code>kubectl set env daemonset/ovs-ovn -n kube-system TUNNEL_TYPE=stt\n\nkubectl delete pod -n kube-system -lapp=ovs\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/security-group/","title":"SecurityGroup Usage","text":"<p>Kube-OVN has supported the configuration of security-groups, and the CRD used to configure security-groups is SecurityGroup.</p>"},{"location":"en/advance/security-group/#securitygroup-example","title":"SecurityGroup Example","text":"<pre><code>apiVersion: kubeovn.io/v1\nkind: SecurityGroup\nmetadata:\nname: sg-example\nspec:\nallowSameGroupTraffic: true\negressRules:\n- ipVersion: ipv4\npolicy: allow\npriority: 1\nprotocol: all\nremoteAddress: 10.16.0.13/16\nremoteType: address\ningressRules:\n- ipVersion: ipv4\npolicy: deny\npriority: 1\nprotocol: icmp\nremoteAddress: 10.16.0.14/16\nremoteType: address\n</code></pre> <p>The specific meaning of each field of the SecurityGroup can be found in the Kube-OVN API Reference.</p> <p>Pods bind security-groups by adding annotations, two annotations are used.</p> <pre><code>    ovn.kubernetes.io/port_security: \"true\"\novn.kubernetes.io/security_groups: sg-example\n</code></pre>"},{"location":"en/advance/security-group/#caution","title":"Caution","text":"<ul> <li> <p>Security-groups are finally restricted by setting ACL rules, and as mentioned in the OVN documentation, if two ACL rules match with the same priority, it is uncertain which ACL will actually work. Therefore, when setting up security-group rules, you need to be careful to differentiate the priority.</p> </li> <li> <p>When adding a security-group, it is important to know what restrictions are being added. As a CNI, Kube-OVN will perform a Pod-to-Gateway connectivity test after creating a Pod.</p> </li> </ul>"},{"location":"en/advance/security-group/#actual-test","title":"Actual test","text":"<p>Create a Pod using the following yaml, and specify the security-group in the annotation for the pod.</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nlabels:\napp: static\nannotations:\novn.kubernetes.io/port_security: 'true'\novn.kubernetes.io/security_groups: 'sg-example'\nname: sg-test-pod\nnamespace: default\nspec:\nnodeName: kube-ovn-worker\ncontainers:\n- image: docker.io/library/nginx:alpine\nimagePullPolicy: IfNotPresent\nname: qatest\n</code></pre> <p>The actual test results show as follows:</p> <pre><code># kubectl get pod -o wide\nNAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES\nsg-test-pod 0/1 ContainerCreating 0 5h32m &lt;none&gt; kube-ovn-worker &lt;none&gt; &lt;none&gt;\ntest-99fff7f86-52h9r 1/1 Running 0 5h41m 10.16.0.14 kube-ovn-control-plane &lt;none&gt; &lt;none&gt;\ntest-99fff7f86-qcgjw 1/1 Running 0 5h43m 10.16.0.13 kube-ovn-worker &lt;none&gt; &lt;none&gt;\n</code></pre> <p>Execute <code>kubectl describe pod</code> to see information about the pod, and you can see the error message:</p> <pre><code># kubectl describe pod sg-test-pod\nName: sg-test-pod\nNamespace: default\nPriority: 0\nNode: kube-ovn-worker/172.18.0.2\nStart Time: Tue, 28 Feb 2023 10:29:36 +0800\nLabels: app=static\nAnnotations: ovn.kubernetes.io/allocated: true\novn.kubernetes.io/cidr: 10.16.0.0/16\n              ovn.kubernetes.io/gateway: 10.16.0.1\n              ovn.kubernetes.io/ip_address: 10.16.0.15\n              ovn.kubernetes.io/logical_router: ovn-cluster\n              ovn.kubernetes.io/logical_switch: ovn-default\n              ovn.kubernetes.io/mac_address: 00:00:00:FA:17:97\n              ovn.kubernetes.io/pod_nic_type: veth-pair\n              ovn.kubernetes.io/port_security: true\novn.kubernetes.io/routed: true\novn.kubernetes.io/security_groups: sg-allow-reject\nStatus: Pending\nIP:\nIPs: &lt;none&gt;\n-\n- -\n- -\nEvents:\n  Type Reason Age From Message\n  ---- ------ ---- ---- -------\n  Warning FailedCreatePodSandBox 5m3s (x70 over 4h59m) kubelet (combined from similar events): Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox \"40636e0c7f1ade5500fa958486163d74f2e2300051a71522a9afd7ba0538afb6\": plugin type=\"kube-ovn\" failed ( add): RPC failed; request ip return 500 configure nic failed 10.16.0.15 network not ready after 200 ping 10.16.0.1\n</code></pre> <p>Modify the rules for the security group to add access rules to the gateway, refer to the following:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: SecurityGroup\nmetadata:\nname: sg-gw-both\nspec:\nallowSameGroupTraffic: true\negressRules:\n- ipVersion: ipv4\npolicy: allow\npriority: 2\nprotocol: all\nremoteAddress: 10.16.0.13/16\nremoteType: address\n- ipVersion: ipv4\npolicy: allow\npriority: 1\nprotocol: all\nremoteAddress: 10.16.0.1/16\nremoteType: address\ningressRules:\n- ipVersion: ipv4\npolicy: deny\npriority: 2\nprotocol: icmp\nremoteAddress: 10.16.0.14/16\nremoteType: address\n- ipVersion: ipv4\npolicy: allow\npriority: 1\nprotocol: icmp\nremoteAddress: 10.16.0.1/16\nremoteType: address\n</code></pre> <p>In the inbound and outbound rules respectively, add a rule to allow access to the gateway, and set the rule to have the highest priority.</p> <p>Deploying with the following yaml to bind security group, confirm that the Pod is operational:</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nlabels:\napp: static\nannotations:\novn.kubernetes.io/port_security: 'true'\novn.kubernetes.io/security_groups: 'sg-gw-both'\nname: sg-gw-both\nnamespace: default\nspec:\nnodeName: kube-ovn-worker\ncontainers:\n- image: docker.io/library/nginx:alpine\nimagePullPolicy: IfNotPresent\nname: qatest\n</code></pre> <p>To view Pod information after deployment:</p> <pre><code># kubectl get pod -o wide\nNAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES\nsg-test-pod 0/1 ContainerCreating 0 5h41m &lt;none&gt; kube-ovn-worker &lt;none&gt; &lt;none&gt;\nsg-gw-both 1/1 Running 0 5h37m 10.16.0.19 kube-ovn-worker &lt;none&gt; &lt;none&gt;\n</code></pre> <p>So for the use of security groups, be particularly clear about the effect of the added restriction rules. If it is simply to restrict traffic access, consider using a network policy instead.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/vip/","title":"VIP Reservation","text":"<p>In some scenarios we want to dynamically reserve part of the IP but not assign it to Pods but to other infrastructure e.g:</p> <ul> <li>Kubernetes nested Kubernetes scenarios where the upper Kubernetes uses the Underlay network take up the available addresses of the underlying Subnet.</li> <li>LB or other network infrastructure requires the use of an IP within a Subnet.</li> </ul>"},{"location":"en/advance/vip/#create-random-address-vip","title":"Create Random Address VIP","text":"<p>If you just want to set aside a number of IPs and have no requirement for the IP addresses themselves, you can use the following yaml to create them:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Vip\nmetadata:\nname: vip-dynamic-01\nspec:\nsubnet: ovn-default\ntype: \"\"\n</code></pre> <ul> <li><code>subnet</code>: reserve the IP from this Subnet.</li> <li><code>type</code>: Currently, two types are supported. If the value is empty, it indicates that it is only used for occupying ip addresses of ipam. <code>switch_lb_vip</code> The front-end vip address and back-end ip address of the switch lb must be on the same subnet.</li> </ul> <p>Query the VIP after creation.</p> <pre><code># kubectl get vip\nNAME             V4IP         PV4IP   MAC                 PMAC   V6IP   PV6IP   SUBNET        READY\nvip-dynamic-01   10.16.0.12           00:00:00:F0:DB:25                         ovn-default   true\n</code></pre> <p>It can be seen that the VIP is assigned the IP address <code>10.16.0.12</code>, which can later be used by other network infrastructures.</p>"},{"location":"en/advance/vip/#create-a-fixed-address-vip","title":"Create a fixed address VIP","text":"<p>The IP address of the reserved VIP can be fixed using the following yaml:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Vip\nmetadata:\nname: static-vip01\nspec:\nsubnet: ovn-default V4ip: \"10.16.0.121\"\n</code></pre> <ul> <li><code>subnet</code>: reserve the IP from this Subnet.</li> <li><code>V4ip</code>: A fixed-assigned IP address that should within the CIDR range of <code>subnet</code>.</li> </ul> <p>Query the VIP after creation:</p> <pre><code># kubectl get vip\nNAME             V4IP         PV4IP   MAC                 PMAC   V6IP   PV6IP   SUBNET        READY\nstatic-vip01   10.16.0.121           00:00:00:F0:DB:26                         ovn-default   true\n</code></pre> <p>It can be seen that the VIP has been assigned the expected IP address.</p>"},{"location":"en/advance/vip/#statefulset-kubevirt-vm-keep-vip","title":"StatefulSet &amp; Kubevirt VM keep VIP","text":"<p>Specify for <code>StatefulSet</code> and <code>VM</code> resources, these Pods their owned will reuse the VIP when these Pods recreating.</p> <p>VM keep VIP must be enable the <code>keep-vm-ip</code> param in <code>kube-ovn-controller</code>. Refer Kubevirt VM Fixed Address Settings</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/vpc-internal-lb/","title":"Customize VPC Internal Load Balancing","text":"<p>The Service provided by Kubernetes can be used for load balancing within the cluster. However, there are several issues with using Service as internal load balancing in customize VPC mode:</p> <ol> <li>The Service IP range is a cluster resource, shared by all customize VPCs, and cannot overlap.</li> <li>Users cannot set internal load balancing IP addresses according to their own preferences.</li> </ol> <p>To address the above issues, Kube OVN introduced the <code>SwitchLBRule</code> CRD in 1.11, allowing users to set internal load balancing rules within customize VPCs.</p> <p><code>SwitchLBRule</code> support the following two ways to set internal load balancing rules within a customize VPC.</p>"},{"location":"en/advance/vpc-internal-lb/#automatically-generate-load-balancing-rules-by-selector","title":"Automatically Generate Load Balancing Rules by <code>Selector</code>","text":"<p>Load balancing rules can be generated by <code>selector</code> automatic association with <code>pod</code> configuration through <code>label</code>.</p> <p>example of <code>SwitchLBRule</code> is as follows:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: SwitchLBRule\nmetadata:\nname:  cjh-slr-nginx\nspec:\nvip: 1.1.1.1\nsessionAffinity: ClientIP\nnamespace: default\nselector:\n- app:nginx\nports:\n- name: dns\nport: 8888\ntargetPort: 80\nprotocol: TCP\n</code></pre> <ul> <li> <p>usage of <code>selector</code>, <code>sessionAffinity</code>, and <code>port</code> is the same as Kubernetes Service.</p> </li> <li> <p><code>vip</code>\uff1acustomize load balancing IP address.</p> </li> <li> <p><code>namespace</code>\uff1anamespace of the <code>pod</code> selected by <code>selector</code>.</p> <p>Kube OVN will determine the VPC of the selected <code>pod</code> based on the <code>SwitchLBRule</code> definition and set the corresponding L2 LB.</p> </li> </ul>"},{"location":"en/advance/vpc-internal-lb/#manually-defined-load-balancing-rules-by-endpoints","title":"Manually Defined Load Balancing Rules by <code>Endpoints</code>","text":"<p>Load balancing rules can be customized configured by   <code>endpoints</code>, to support scenarios where load balancing rules cannot be automatically generated through <code>selector</code>.  For example, the load balancing backend is <code>vm</code> created by <code>kubevirt</code>.</p> <p>example of <code>SwitchLBRule</code> is as follows:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: SwitchLBRule\nmetadata:\nname:  cjh-slr-nginx\nspec:\nvip: 1.1.1.1\nsessionAffinity: ClientIP\nnamespace: default\nendpoints:\n- 192.168.0.101\n- 192.168.0.102\n- 192.168.0.103\nports:\n- name: dns\nport: 8888\ntargetPort: 80\nprotocol: TCP\n</code></pre> <ul> <li> <p>usage of <code>sessionAffinity</code>, and <code>port</code> is the same as Kubernetes Service.</p> </li> <li> <p><code>vip</code>\uff1acustomize load balancing IP address.</p> </li> <li> <p><code>namespace</code>\uff1anamespace of the <code>pod</code> selected by <code>selector</code>.</p> </li> <li> <p><code>endpoints</code>\uff1aload balancing backend IP list.</p> </li> </ul> <p>attention\uff1aIf both <code>selector</code> and <code>endpoints</code> are configured, the <code>selector</code> configuration will be automatically ignored.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/vpc-peering/","title":"VPC Peering","text":"<p>VPC peering provides a mechanism for bridging two VPC networks through logical routes so that workloads within two VPCs can access each other through private addresses as if they were on the same private network, without the need for NAT forwarding through a gateway.</p>"},{"location":"en/advance/vpc-peering/#prerequisites","title":"Prerequisites","text":"<ol> <li>This feature is only available for customized VPCs.</li> <li>To avoid route overlap the subnet CIDRs within the two VPCs cannot overlap.</li> <li>Currently, only interconnection of two VPCs is supported.</li> </ol>"},{"location":"en/advance/vpc-peering/#usage","title":"Usage","text":"<p>First create two non-interconnected VPCs with one Subnet under each VPC, and the CIDRs of the Subnets do not overlap with each other.</p> <pre><code>kind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: vpc-1\nspec: {}\n---\nkind: Subnet\napiVersion: kubeovn.io/v1\nmetadata:\nname: net1\nspec:\nvpc: vpc-1\ncidrBlock: 10.0.0.0/16\n---\nkind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: vpc-2\nspec: {}\n---\nkind: Subnet\napiVersion: kubeovn.io/v1\nmetadata:\nname: net2\nspec:\nvpc: vpc-2\ncidrBlock: 172.31.0.0/16\n</code></pre> <p>Add <code>vpcPeerings</code> and the corresponding static routes within each VPC:</p> <pre><code>kind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: vpc-1\nspec: vpcPeerings:\n- remoteVpc: vpc-2\nlocalConnectIP: 169.254.0.1/30\nstaticRoutes:\n- cidr: 172.31.0.0/16\nnextHopIP: 169.254.0.2\npolicy: policyDst\n---\nkind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: vpc-2\nspec:\nvpcPeerings:\n- remoteVpc: vpc-1\nlocalConnectIP: 169.254.0.2/30\nstaticRoutes:\n- cidr: 10.0.0.0/16\nnextHopIP: 169.254.0.1\npolicy: policyDst\n</code></pre> <ul> <li><code>remoteVpc</code>: The name of another peering VPC.</li> <li><code>localConnectIP</code>: As the IP address and CIDR of the interconnection endpoint. Note that both IPs should belong to the same CIDR and should not conflict with existing subnets.</li> <li><code>cidr</code>\uff1aCIDR of the peering Subnet.</li> <li><code>nextHopIP</code>\uff1aThe <code>localConnectIP</code> on the other end of the peering VPC.</li> </ul> <p>Create Pods under the two Subnets</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nannotations:\novn.kubernetes.io/logical_switch: net1\nname: vpc-1-pod\nspec:\ncontainers:\n- name: vpc-1-pod\nimage: docker.io/library/nginx:alpine\n---\napiVersion: v1\nkind: Pod\nmetadata:\nannotations:\novn.kubernetes.io/logical_switch: net2\nname: vpc-2-pod\nspec:\ncontainers:\n- name: vpc-2-pod\nimage: docker.io/library/nginx:alpine\n</code></pre> <p>Test the network connectivity</p> <pre><code># kubectl exec -it vpc-1-pod -- ping $(kubectl get pod vpc-2-pod -o jsonpath='{.status.podIP}')\nPING 172.31.0.2 (172.31.0.2): 56 data bytes\n64 bytes from 172.31.0.2: seq=0 ttl=62 time=0.655 ms\n64 bytes from 172.31.0.2: seq=1 ttl=62 time=0.086 ms\n64 bytes from 172.31.0.2: seq=2 ttl=62 time=0.098 ms\n^C\n--- 172.31.0.2 ping statistics ---\n3 packets transmitted, 3 packets received, 0% packet loss\nround-trip min/avg/max = 0.086/0.279/0.655 ms\n# kubectl exec -it vpc-2-pod -- ping $(kubectl get pod vpc-1-pod -o jsonpath='{.status.podIP}')\nPING 10.0.0.2 (10.0.0.2): 56 data bytes\n64 bytes from 10.0.0.2: seq=0 ttl=62 time=0.594 ms\n64 bytes from 10.0.0.2: seq=1 ttl=62 time=0.093 ms\n64 bytes from 10.0.0.2: seq=2 ttl=62 time=0.088 ms\n^C\n--- 10.0.0.2 ping statistics ---\n3 packets transmitted, 3 packets received, 0% packet loss\nround-trip min/avg/max = 0.088/0.258/0.594 ms\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/windows/","title":"Windows Support","text":"<p>Kube-OVN supports Kubernetes cluster networks that include Windows system nodes, allowing unified containers network management.</p>"},{"location":"en/advance/windows/#prerequisites","title":"Prerequisites","text":"<ul> <li>Read Adding Windows nodes to add Windows nodes.</li> <li>Windows nodes must have the KB4489899 patch installed for Overlay/VXLAN networks to work properly, and it is recommended to update your system to the latest version.</li> <li>Hyper-V and management tools must be installed on the Windows node.</li> <li>Due to Windows restrictions tunnel encapsulation can only be used in Vxlan mode.</li> <li>SSL, IPv6, dual-stack, QoS features are not supported at this time.</li> <li>Dynamic subnet and dynamic tunnel interface are not supported at this time. You need to create the subnet and select the network interface before installing the Windows node.</li> <li>Multiple <code>ProviderNetwork</code>s are not supported, and the bridge interface configuration cannot be dynamically adjusted.</li> </ul>"},{"location":"en/advance/windows/#install-ovs-on-windows","title":"Install OVS on Windows","text":"<p>Due to some issues with upstream OVN and OVS support for Windows containers, a modified installation package provided by Kube-OVN is required.</p> <p>Use the following command to enable the <code>TESTSIGNING</code> startup item on the Windows node, which requires a system reboot to take effect.</p> <pre><code>bcdedit /set LOADOPTIONS DISABLE_INTEGRITY_CHECKS\nbcdedit /set TESTSIGNING ON\nbcdedit /set nointegritychecks ON\n</code></pre> <p>Download Windows package on Windows node and install.</p> <p>Confirm that the service is running properly after installation:</p> <pre><code>PS &gt; Get-Service | findstr ovs\nRunning  ovsdb-server  Open vSwitch DB Service\nRunning  ovs-vswitchd  Open vSwitch Service\n</code></pre>"},{"location":"en/advance/windows/#install-kube-ovn","title":"Install Kube-OVN","text":"<p>Download the installation script in the Windows node install.ps1.</p> <p>Add relevant parameters and run:</p> <pre><code>.\\install.ps1 -KubeConfig C:\\k\\admin.conf -ApiServer https://192.168.140.180:6443 -ServiceCIDR 10.96.0.0/12\n</code></pre> <p>By default, Kube-OVN uses the NIC where the node IP is located as the tunnel interface. If you need to use another NIC, you need to add the specified annotation to the Node before installation, e.g. <code>ovn.kubernetes.io/tunnel_interface=Ethernet1</code>.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/with-bgp/","title":"BGP Support","text":"<p>Kube-OVN supports broadcasting the IP address of the Pod or Subnet to the outside world via BGP protocol, so that the outside world can access the Pod directly through the Pod IP. To use this feature, you need to install <code>kube-ovn-speaker</code> on specific nodes and add the corresponding annotation to the Pod or Subnet that needs to be exposed to the outside world.</p>"},{"location":"en/advance/with-bgp/#install-kube-ovn-speaker","title":"Install kube-ovn-speaker","text":"<p><code>kube-ovn-speaker</code> use GoBGP to publish routing information to the outside world and set the next-hop route to itself.</p> <p>Since the node where <code>kube-ovn-speaker</code> is deployed needs to carry return traffic, specific labeled nodes need to be selected for deployment:</p> <pre><code>kubectl label nodes speaker-node-1 ovn.kubernetes.io/bgp=true\nkubectl label nodes speaker-node-2 ovn.kubernetes.io/bgp=true\n</code></pre> <p>When there are multiple instances of kube-ovn-speaker, each of them will publish routes to the outside world, the upstream router needs to support multi-path ECMP.</p> <p>Download the corresponding yaml:</p> <pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/yamls/speaker.yaml\n</code></pre> <p>Modify the corresponding configuration in yaml:</p> <pre><code>--neighbor-address=10.32.32.1\n--neighbor-as=65030\n--cluster-as=65000\n</code></pre> <ul> <li><code>neighbor-address</code>: The address of the BGP Peer, usually the router gateway address.</li> <li><code>neighbor-as</code>: The AS number of the BGP Peer.</li> <li><code>cluster-as</code>: The AS number of the container network.</li> </ul> <p>Deploy yaml:</p> <pre><code>kubectl apply -f speaker.yaml\n</code></pre>"},{"location":"en/advance/with-bgp/#publish-podsubnet-routes","title":"Publish Pod/Subnet Routes","text":"<p>To use BGP for external routing, first set <code>natOutgoing</code> to <code>false</code> for the corresponding Subnet to allow the Pod IP to enter the underlying network directly.</p> <p>Add annotation to publish routes:</p> <pre><code>kubectl annotate pod sample ovn.kubernetes.io/bgp=true\nkubectl annotate subnet ovn-default ovn.kubernetes.io/bgp=true\n</code></pre> <p>Delete annotation to disable the publishing:</p> <pre><code>kubectl annotate pod perf-ovn-xzvd4 ovn.kubernetes.io/bgp-\nkubectl annotate subnet ovn-default ovn.kubernetes.io/bgp-\n</code></pre>"},{"location":"en/advance/with-bgp/#bgp-advance-options","title":"BGP Advance Options","text":"<p><code>kube-ovn-speaker</code> supports more BGP parameters for advanced configuration, which can be adjusted by users according to their network environment:</p> <ul> <li><code>announce-cluster-ip</code>: Whether to publish Service routes to the public, default is <code>false</code>.</li> <li><code>auth-password</code>: The access password for the BGP peer.</li> <li><code>holdtime</code>: The heartbeat detection time between BGP neighbors. Neighbors with no messages after the change time will be removed, the default is 90 seconds.</li> <li><code>graceful-restart</code>: Whether to enable BGP Graceful Restart.</li> <li><code>graceful-restart-time</code>: BGP Graceful restart time refer to RFC4724 3.</li> <li><code>graceful-restart-deferral-time</code>: BGP Graceful restart deferral time refer to RFC4724 4.1.</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/with-cilium/","title":"Integration with Cilium","text":"<p>Cilium is an eBPF-based networking and security component. Kube-OVN uses the CNI Chaining mode to enhance existing features. Users can use both the rich network abstraction capabilities of Kube-OVN and the monitoring and security capabilities that come with eBPF.</p> <p>By integrating Cilium, Kube-OVN users can have the following gains:</p> <ul> <li>Richer and more efficient security policies.</li> <li>Hubble-based monitoring and UI.</li> </ul> <p></p>"},{"location":"en/advance/with-cilium/#prerequisites","title":"Prerequisites","text":"<ol> <li>Linux kernel version above 4.19 or other compatible kernel for full eBPF capability support.</li> <li>Install Helm in advance to prepare for the installation of Cilium, please refer to Installing Helm to deploy Helm.</li> </ol>"},{"location":"en/advance/with-cilium/#configure-kube-ovn","title":"Configure Kube-OVN","text":"<p>In order to fully utilize the security capabilities of Cilium, you need to disable the <code>networkpolicy</code> feature within Kube-OVN and adjust the CNI configuration priority.</p> <p>Change the following variables in the <code>install.sh</code> script:</p> <pre><code>ENABLE_NP=false\nCNI_CONFIG_PRIORITY=10\n</code></pre> <p>If the deployment is complete, you can adjust the args of <code>kube-ovn-controller</code>:</p> <pre><code>args:\n- --enable-np=false\n</code></pre> <p>Modify the <code>kube-ovn-cni</code> args to adjust the CNI configuration priority:</p> <pre><code>args:\n- --cni-conf-name=10-kube-ovn.conflist\n</code></pre> <p>Adjust the Kube-OVN cni configuration name on each node:</p> <pre><code>mv /etc/cni/net.d/01-kube-ovn.conflist /etc/cni/net.d/10-kube-ovn.conflist\n</code></pre>"},{"location":"en/advance/with-cilium/#deploy-cilium","title":"Deploy Cilium","text":"<p>Create the <code>chaining.yaml</code> configuration file to use Cilium's <code>generic-veth</code> mode:</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: cni-configuration\nnamespace: kube-system\ndata:\ncni-config: |-\n{\n\"name\": \"generic-veth\",\n\"cniVersion\": \"0.3.1\",\n\"plugins\": [\n{\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\",\n\"ipam\": {\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\"\n}\n},\n{\n\"type\": \"portmap\",\n\"snat\": true,\n\"capabilities\": {\"portMappings\": true}\n},\n{\n\"type\": \"cilium-cni\"\n}\n]\n}\n</code></pre> <p>Installation the chaining config:</p> <pre><code>kubectl apply -f chaining.yaml\n</code></pre> <p>Deploying Cilium with Helm:</p> <pre><code>helm repo add cilium https://helm.cilium.io/\nhelm install cilium cilium/cilium --version 1.11.6 \\\n--namespace kube-system \\\n--set cni.chainingMode=generic-veth \\\n--set cni.customConf=true \\\n--set cni.configMap=cni-configuration \\\n--set tunnel=disabled \\\n--set enableIPv4Masquerade=false \\\n--set enableIdentityMark=false </code></pre> <p>Confirm that the Cilium installation was successful:</p> <pre><code># cilium  status\n/\u00af\u00af\\\n/\u00af\u00af\\__/\u00af\u00af\\    Cilium:         OK\n \\__/\u00af\u00af\\__/    Operator:       OK\n /\u00af\u00af\\__/\u00af\u00af\\    Hubble:         disabled\n \\__/\u00af\u00af\\__/    ClusterMesh:    disabled\n    \\__/\n\nDaemonSet         cilium             Desired: 2, Ready: 2/2, Available: 2/2\nDeployment        cilium-operator    Desired: 2, Ready: 2/2, Available: 2/2\nContainers:       cilium             Running: 2\ncilium-operator    Running: 2\nCluster Pods:     8/11 managed by Cilium\nImage versions    cilium             quay.io/cilium/cilium:v1.10.5@sha256:0612218e28288db360c63677c09fafa2d17edda4f13867bcabf87056046b33bb: 2\ncilium-operator    quay.io/cilium/operator-generic:v1.10.5@sha256:2d2f730f219d489ff0702923bf24c0002cd93eb4b47ba344375566202f56d972: 2\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/with-openstack/","title":"Integration with OpenStack","text":"<p>In some cases, users need to run virtual machines with OpenStack and containers with Kubernetes, and need the network to interoperate between containers and virtual machines and be under a unified control plane. If the OpenStack Neutron side also uses OVN as the underlying network, then Kube-OVN can use either cluster interconnection or shared underlying OVN to connect the OpenStack and Kubernetes networks.</p>"},{"location":"en/advance/with-openstack/#cluster-interconnection","title":"Cluster Interconnection","text":"<p>This pattern is similar to Cluster Inter-Connection with OVN-IC to connect two Kubernetes cluster networks, except that the two ends of the cluster are replaced with OpenStack and Kubernetes\u3002</p>"},{"location":"en/advance/with-openstack/#prerequisites","title":"Prerequisites","text":"<ol> <li>The subnet CIDRs within OpenStack and Kubernetes cannot overlap with each other in auto-route mode.</li> <li>A set of machines needs to exist that can be accessed by each cluster over the network and used to deploy controllers that interconnect across clusters.</li> <li>Each cluster needs to have a set of machines that can access each other across clusters via IP as the gateway nodes.</li> <li>This solution only connects to the Kubernetes default subnet with selected VPC in OpenStack.</li> </ol>"},{"location":"en/advance/with-openstack/#deploy-ovn-ic-db","title":"Deploy OVN-IC DB","text":"<p>Start the <code>OVN-IC</code> DB with the following command:</p> <pre><code>docker run --name=ovn-ic-db -d --network=host -v /etc/ovn/:/etc/ovn -v /var/run/ovn:/var/run/ovn -v /var/log/ovn:/var/log/ovn kubeovn/kube-ovn:v1.13.0 bash start-ic-db.sh\n</code></pre>"},{"location":"en/advance/with-openstack/#kubernetes-side-operations","title":"Kubernetes Side Operations","text":"<p>Create <code>ovn-ic-config</code> ConfigMap in <code>kube-system</code> Namespace \uff1a</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: ovn-ic-config\nnamespace: kube-system\ndata:\nenable-ic: \"true\"\naz-name: \"az1\" ic-db-host: \"192.168.65.3\"\nic-nb-port: \"6645\" ic-sb-port: \"6646\"\ngw-nodes: \"az1-gw\"\nauto-route: \"true\"\n</code></pre> <ul> <li><code>enable-ic</code>: Whether to enable cluster interconnection.</li> <li><code>az-name</code>: Distinguish the cluster names of different clusters, each interconnected cluster needs to be different.</li> <li><code>ic-db-host</code>: Address of the node where the <code>OVN-IC</code> DB is deployed.</li> <li><code>ic-nb-port</code>: <code>OVN-IC</code> Northbound Database port, default 6645.</li> <li><code>ic-sb-port</code>: <code>OVN-IC</code> Southbound Database port, default 6645.</li> <li><code>gw-nodes</code>: The name of the nodes in the cluster interconnection that takes on the work of the gateways, separated by commas.</li> <li><code>auto-route</code>: Whether to automatically publish and learn routes.</li> </ul>"},{"location":"en/advance/with-openstack/#openstack-side-operations","title":"OpenStack Side Operations","text":"<p>Create logical routers that interconnect with Kubernetes:</p> <pre><code># openstack router create router0\n# openstack router list\n+--------------------------------------+---------+--------+-------+----------------------------------+\n| ID                                   | Name    | Status | State | Project                          |\n+--------------------------------------+---------+--------+-------+----------------------------------+\n| d5b38655-249a-4192-8046-71aa4d2b4af1 | router0 | ACTIVE | UP    | 98a29ab7388347e7b5ff8bdd181ba4f9 |\n+--------------------------------------+---------+--------+-------+----------------------------------+\n</code></pre> <p>Set the availability zone name in the OVN northbound database within OpenStack, which needs to be different from the other interconnected clusters:</p> <pre><code>ovn-nbctl set NB_Global . name=op-az\n</code></pre> <p>Start the <code>OVN-IC</code> controller at a node that has access to the <code>OVN-IC</code> DB:</p> <pre><code>/usr/share/ovn/scripts/ovn-ctl --ovn-ic-nb-db=tcp:192.168.65.3:6645 \\\n--ovn-ic-sb-db=tcp:192.168.65.3:6646 \\\n--ovn-northd-nb-db=unix:/run/ovn/ovnnb_db.sock \\\n--ovn-northd-sb-db=unix:/run/ovn/ovnsb_db.sock \\\nstart_ic\n</code></pre> <ul> <li><code>ovn-ic-nb-db</code>\uff0c<code>ovn-ic-sb-db</code>: OVN-IC Northbound database and southbound database addresses.</li> <li><code>ovn-northd-nb-db</code>\uff0c <code>ovn-northd-sb-db</code>: Current cluster OVN northbound database and southbound data address.</li> </ul> <p>Configuration gateway nodes:</p> <pre><code>ovs-vsctl set open_vswitch . external_ids:ovn-is-interconn=true\n</code></pre> <p>The next step is to create a logical topology by operating the OVN in OpenStack.</p> <p>Connect the <code>ts</code> interconnect switch and the <code>router0</code> logical router, and set the relevant rules:</p> <pre><code>ovn-nbctl lrp-add router0 lrp-router0-ts 00:02:ef:11:39:4f 169.254.100.73/24\novn-nbctl lsp-add ts lsp-ts-router0 -- lsp-set-addresses lsp-ts-router0 router \\\n-- lsp-set-type lsp-ts-router0 router \\\n-- lsp-set-options lsp-ts-router0  router-port=lrp-router0-ts\novn-nbctl lrp-set-gateway-chassis lrp-router0-ts {gateway chassis} 1000\novn-nbctl set NB_Global . options:ic-route-adv=true options:ic-route-learn=true\n</code></pre> <p>Verify that OpenStack has learned the Kubernetes routing rules:</p> <pre><code># ovn-nbctl lr-route-list router0\nIPv4 Routes\n                10.0.0.22            169.254.100.34 dst-ip (learned)\n10.16.0.0/16            169.254.100.34 dst-ip (learned)\n</code></pre> <p>Next, you can create a virtual machine under the <code>router0</code> network to verify that it can interconnect with Pods under Kubernetes.</p>"},{"location":"en/advance/with-openstack/#shared-underlay-ovn","title":"Shared Underlay OVN","text":"<p>In this scenario, OpenStack and Kubernetes share the same OVN, so concepts such as VPC and Subnet can be pulled together for better control and interconnection.</p> <p>In this mode we deploy the OVN normally using Kube-OVN, and OpenStack modifies the Neutron configuration to connect to the same OVN DB. OpenStack requires networking-ovn as a Neutron backend implementation.</p>"},{"location":"en/advance/with-openstack/#neutron-modification","title":"Neutron Modification","text":"<p>Modify the Neutron configuration file <code>/etc/neutron/plugins/ml2/ml2_conf.ini</code>\uff1a</p> <pre><code>[ovn]\n...\novn_nb_connection = tcp:[192.168.137.176]:6641,tcp:[192.168.137.177]:6641,tcp:[192.168.137.178]:6641\novn_sb_connection = tcp:[192.168.137.176]:6642,tcp:[192.168.137.177]:6642,tcp:[192.168.137.178]:6642\novn_l3_scheduler = OVN_L3_SCHEDULER\n</code></pre> <ul> <li><code>ovn_nb_connection</code>\uff0c <code>ovn_sb_connection</code>: The address needs to be changed to the address of the <code>ovn-central</code> nodes deployed by Kube-OVN.</li> </ul> <p>Modify the OVS configuration for each node:</p> <pre><code>ovs-vsctl set open . external-ids:ovn-remote=tcp:[192.168.137.176]:6642,tcp:[192.168.137.177]:6642,tcp:[192.168.137.178]:6642\novs-vsctl set open . external-ids:ovn-encap-type=geneve\novs-vsctl set open . external-ids:ovn-encap-ip=192.168.137.200\n</code></pre> <ul> <li><code>external-ids:ovn-remote</code>: The address needs to be changed to the address of the <code>ovn-central</code> nodes deployed by Kube-OVN.</li> <li><code>ovn-encap-ip</code>: Change to the IP address of the current node.</li> </ul>"},{"location":"en/advance/with-openstack/#using-openstack-internal-resources-in-kubernetes","title":"Using OpenStack Internal Resources in Kubernetes","text":"<p>The next section describes how to query OpenStack's network resources in Kubernetes and create Pods in the subnet from OpenStack.</p> <p>Query the existing network resources in OpenStack for the following resources that have been pre-created.</p> <pre><code># openstack router list\n+--------------------------------------+---------+--------+-------+----------------------------------+\n| ID                                   | Name    | Status | State | Project                          |\n+--------------------------------------+---------+--------+-------+----------------------------------+\n| 22040ed5-0598-4f77-bffd-e7fd4db47e93 | router0 | ACTIVE | UP    | 62381a21d569404aa236a5dd8712449c |\n+--------------------------------------+---------+--------+-------+----------------------------------+\n# openstack network list\n+--------------------------------------+----------+--------------------------------------+\n| ID                                   | Name     | Subnets                              |\n+--------------------------------------+----------+--------------------------------------+\n| cd59e36a-37db-4c27-b709-d35379a7920f | provider | 01d73d9f-fdaa-426c-9b60-aa34abbfacae |\n+--------------------------------------+----------+--------------------------------------+\n# openstack subnet list\n+--------------------------------------+-------------+--------------------------------------+----------------+\n| ID                                   | Name        | Network                              | Subnet         |\n+--------------------------------------+-------------+--------------------------------------+----------------+\n| 01d73d9f-fdaa-426c-9b60-aa34abbfacae | provider-v4 | cd59e36a-37db-4c27-b709-d35379a7920f | 192.168.1.0/24 |\n+--------------------------------------+-------------+--------------------------------------+----------------+\n# openstack server list\n+--------------------------------------+-------------------+--------+-----------------------+--------+--------+\n| ID                                   | Name              | Status | Networks              | Image  | Flavor |\n+--------------------------------------+-------------------+--------+-----------------------+--------+--------+\n| 8433d622-a8d6-41a7-8b31-49abfd64f639 | provider-instance | ACTIVE | provider=192.168.1.61 | ubuntu | m1     |\n+--------------------------------------+-------------------+--------+-----------------------+--------+--------+\n</code></pre> <p>On the Kubernetes side, query the VPC resources from OpenStack:</p> <pre><code># kubectl get vpc\nNAME                                           STANDBY   SUBNETS\nneutron-22040ed5-0598-4f77-bffd-e7fd4db47e93   true      [\"neutron-cd59e36a-37db-4c27-b709-d35379a7920f\"]\novn-cluster                                    true      [\"join\",\"ovn-default\"]\n</code></pre> <p><code>neutron-22040ed5-0598-4f77-bffd-e7fd4db47e93</code> is the VPC resources synchronized from OpenStack.</p> <p>Next, you can create Pods and run them according to Kube-OVN's native VPC and Subnet operations.</p> <p>Bind VPC, Subnet to Namespace <code>net2</code> and create Pod:</p> <pre><code>apiVersion: v1\nkind: Namespace\nmetadata:\nname: net2\n---\napiVersion: kubeovn.io/v1\nkind: Vpc\nmetadata:\ncreationTimestamp: \"2021-06-20T13:34:11Z\"\ngeneration: 2\nlabels:\novn.kubernetes.io/vpc_external: \"true\"\nname: neutron-22040ed5-0598-4f77-bffd-e7fd4db47e93\nresourceVersion: \"583728\"\nuid: 18d4c654-f511-4def-a3a0-a6434d237c1e\nspec:\nnamespaces:\n- net2\n---\nkind: Subnet\napiVersion: kubeovn.io/v1\nmetadata:\nname: net2\nspec:\nvpc: neutron-22040ed5-0598-4f77-bffd-e7fd4db47e93\nnamespaces:\n- net2\ncidrBlock: 12.0.1.0/24\nnatOutgoing: false\n---\napiVersion: v1\nkind: Pod\nmetadata:\nname: ubuntu\nnamespace: net2\nspec:\ncontainers:\n- image: docker.io/kubeovn/kube-ovn:v1.8.0\ncommand:\n- \"sleep\"\n- \"604800\"\nimagePullPolicy: IfNotPresent\nname: ubuntu\nrestartPolicy: Always\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/with-ovn-ic/","title":"Cluster Inter-Connection with OVN-IC","text":"<p>Kube-OVN supports interconnecting two Kubernetes cluster Pod networks via OVN-IC, and the Pods in the two clusters can communicate directly via Pod IPs . Kube-OVN uses tunnels to encapsulate cross-cluster traffic, allowing container networks to interconnect between two clusters as long as there is a set of IP reachable machines.</p> <p>This mode of multi-cluster interconnection is for Overlay network. For Underlay network, it needs the underlying infrastructure to do the inter-connection work.</p> <p></p>"},{"location":"en/advance/with-ovn-ic/#prerequisites","title":"Prerequisites","text":"<ol> <li>The subnet CIDRs within OpenStack and Kubernetes cannot overlap with each other in auto-interconnect mode.    If there is overlap, you need to refer to the subsequent manual interconnection process, which can only connect non-overlapping Subnets.</li> <li>A set of machines needs to exist that can be accessed by each cluster over the network and used to deploy controllers that interconnect across clusters.</li> <li>Each cluster needs to have a set of machines that can access each other across clusters via IP as the gateway nodes.</li> <li>This solution only connects to the Kubernetes default VPCs.</li> </ol>"},{"location":"en/advance/with-ovn-ic/#deploy-a-single-node-ovn-ic-db","title":"Deploy a single-node OVN-IC DB","text":"<p>Deploy the <code>OVN-IC</code> DB on a machine accessible by <code>kube-ovn-controller</code>, This DB will hold the network configuration information synchronized up from each cluster.</p> <p>An environment deploying <code>docker</code> can start the <code>OVN-IC</code> DB with the following command.</p> <pre><code>docker run --name=ovn-ic-db -d --network=host --privileged -v /etc/ovn/:/etc/ovn -v /var/run/ovn:/var/run/ovn -v /var/log/ovn:/var/log/ovn kubeovn/kube-ovn:v1.13.0 bash start-ic-db.sh\n</code></pre> <p>For deploying a <code>containerd</code> environment instead of <code>docker</code> you can use the following command:</p> <pre><code>ctr -n k8s.io run -d --net-host --privileged --mount=\"type=bind,src=/etc/ovn/,dst=/etc/ovn,options=rbind:rw\" --mount=\"type=bind,src=/var/run/ovn,dst=/var/run/ovn,options=rbind:rw\" --mount=\"type=bind,src=/var/log/ovn,dst=/var/log/ovn,options=rbind:rw\" docker.io/kubeovn/kube-ovn:v1.13.0 ovn-ic-db bash start-ic-db.sh\n</code></pre>"},{"location":"en/advance/with-ovn-ic/#automatic-routing-mode","title":"Automatic Routing Mode","text":"<p>In auto-routing mode, each cluster synchronizes the CIDR information of the Subnet under its own default VPC to <code>OVN-IC</code>, so make sure there is no overlap between the Subnet CIDRs of the two clusters.</p> <p>Create <code>ovn-ic-config</code> ConfigMap in <code>kube-system</code> Namespace:</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: ovn-ic-config\nnamespace: kube-system\ndata:\nenable-ic: \"true\"\naz-name: \"az1\" ic-db-host: \"192.168.65.3\"\nic-nb-port: \"6645\" ic-sb-port: \"6646\"\ngw-nodes: \"az1-gw\"\nauto-route: \"true\"\n</code></pre> <ul> <li><code>enable-ic</code>: Whether to enable cluster interconnection.</li> <li><code>az-name</code>: Distinguish the cluster names of different clusters, each interconnected cluster needs to be different.</li> <li><code>ic-db-host</code>: Address of the node where the <code>OVN-IC</code> DB is deployed.</li> <li><code>ic-nb-port</code>: <code>OVN-IC</code> Northbound Database port, default 6645.</li> <li><code>ic-sb-port</code>: <code>OVN-IC</code> Southbound Database port, default 6645.</li> <li><code>gw-nodes</code>: The name of the nodes in the cluster interconnection that takes on the work of the gateways, separated by commas.</li> <li><code>auto-route</code>: Whether to automatically publish and learn routes.</li> </ul> <p>Note: To ensure the correct operation, the ConfigMap <code>ovn-ic-config</code> is not allowed to be modified. If any parameter needs to be changed, please delete this ConfigMap, modify it and then apply it again.</p> <p>Check if the interconnected logical switch <code>ts</code> has been established in the <code>ovn-ic</code> container with the following command\uff1a</p> <pre><code># ovn-ic-sbctl show\navailability-zone az1\n    gateway deee03e0-af16-4f45-91e9-b50c3960f809\n        hostname: az1-gw\n        type: geneve\n            ip: 192.168.42.145\n        port ts-az1\n            transit switch: ts\n            address: [\"00:00:00:50:AC:8C 169.254.100.45/24\"]\navailability-zone az2\n    gateway e94cc831-8143-40e3-a478-90352773327b\n        hostname: az2-gw\n        type: geneve\n            ip: 192.168.42.149\n        port ts-az2\n            transit switch: ts\n            address: [\"00:00:00:07:4A:59 169.254.100.63/24\"]\n</code></pre> <p>At each cluster observe if logical routes have learned peer routes:</p> <pre><code># kubectl ko nbctl lr-route-list ovn-cluster\nIPv4 Routes\n                10.42.1.1            169.254.100.45 dst-ip (learned)\n10.42.1.3                100.64.0.2 dst-ip\n                10.16.0.2                100.64.0.2 src-ip\n                10.16.0.3                100.64.0.2 src-ip\n                10.16.0.4                100.64.0.2 src-ip\n                10.16.0.6                100.64.0.2 src-ip\n             10.17.0.0/16            169.254.100.45 dst-ip (learned)\n100.65.0.0/16            169.254.100.45 dst-ip (learned)\n</code></pre> <p>Next, you can try <code>ping</code> a Pod IP in Cluster 1 directly from a Pod in Cluster 2 to see if you can work.</p> <p>For a subnet that does not want to automatically publish routes to the other end, you can disable route broadcasting by modifying <code>disableInterConnection</code> in the Subnet spec.</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: no-advertise\nspec:\ncidrBlock: 10.199.0.0/16\ndisableInterConnection: true\n</code></pre>"},{"location":"en/advance/with-ovn-ic/#manual-routing-mode","title":"Manual Routing Mode","text":"<p>For cases where there are overlapping CIDRs between clusters, and you only want to do partial subnet interconnection, you can manually publish subnet routing by following the steps below.</p> <p>Create <code>ovn-ic-config</code> ConfigMap in <code>kube-system</code> Namespace, and set <code>auto-route</code> to <code>false</code>:</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: ovn-ic-config\nnamespace: kube-system\ndata:\nenable-ic: \"true\"\naz-name: \"az1\" ic-db-host: \"192.168.65.3\"\nic-nb-port: \"6645\" ic-sb-port: \"6646\"\ngw-nodes: \"az1-gw\"\nauto-route: \"false\"\n</code></pre> <p>Find the address of the remote logical ports in each cluster separately, for later manual configuration of the route:</p> <pre><code>[root@az1 ~]# kubectl ko nbctl show\nswitch a391d3a1-14a0-4841-9836-4bd930c447fb (ts)\nport ts-az1\n        type: router\n        router-port: az1-ts\n    port ts-az2\n        type: remote\n        addresses: [\"00:00:00:4B:E2:9F 169.254.100.31/24\"]\n\n[root@az2 ~]# kubectl ko nbctl show\nswitch da6138b8-de81-4908-abf9-b2224ec4edf3 (ts)\nport ts-az2\n        type: router\n        router-port: az2-ts\n    port ts-az1\n        type: remote\n        addresses: [\"00:00:00:FB:2A:F7 169.254.100.79/24\"]        </code></pre> <p>The output above shows that the remote address from cluster <code>az1</code> to cluster <code>az2</code> is <code>169.254.100.31</code> and the remote address from <code>az2</code> to <code>az1</code> is <code>169.254.100.79</code>.</p> <p>In this example, the subnet CIDR within cluster <code>az1</code> is <code>10.16.0.0/24</code> and the subnet CIDR within cluster <code>az2</code> is <code>10.17.0.0/24</code>.</p> <p>Set up a route from cluster <code>az1</code> to cluster <code>az2</code> in cluster <code>az1</code>:</p> <pre><code>kubectl ko nbctl lr-route-add ovn-cluster 10.17.0.0/24 169.254.100.31\n</code></pre> <p>Set up a route to cluster <code>az1</code> in cluster <code>az2</code>:</p> <pre><code>kubectl ko nbctl lr-route-add ovn-cluster 10.16.0.0/24 169.254.100.79\n</code></pre>"},{"location":"en/advance/with-ovn-ic/#highly-available-ovn-ic-db-installation","title":"Highly Available OVN-IC DB Installation","text":"<p>A highly available cluster can be formed between <code>OVN-IC</code> DB via the Raft protocol, which requires a minimum of 3 nodes for this deployment model.</p> <p>First start the leader of the <code>OVN-IC</code> DB on the first node.</p> <p>Users deploying a <code>docker</code> environment can use the following command:</p> <pre><code>docker run --name=ovn-ic-db -d --network=host --privileged -v /etc/ovn/:/etc/ovn -v /var/run/ovn:/var/run/ovn -v /var/log/ovn:/var/log/ovn -e LOCAL_IP=\"192.168.65.3\"  -e NODE_IPS=\"192.168.65.3,192.168.65.2,192.168.65.1\"   kubeovn/kube-ovn:v1.13.0 bash start-ic-db.sh\n</code></pre> <p>If you are  using <code>containerd</code> you can use the following command:</p> <pre><code>ctr -n k8s.io run -d --net-host --privileged --mount=\"type=bind,src=/etc/ovn/,dst=/etc/ovn,options=rbind:rw\" --mount=\"type=bind,src=/var/run/ovn,dst=/var/run/ovn,options=rbind:rw\" --mount=\"type=bind,src=/var/log/ovn,dst=/var/log/ovn,options=rbind:rw\"  --env=\"NODE_IPS=\"192.168.65.3,192.168.65.2,192.168.65.1\"\" --env=\"LOCAL_IP=\"192.168.65.3\"\" docker.io/kubeovn/kube-ovn:v1.13.0 ovn-ic-db bash start-ic-db.sh\n</code></pre> <ul> <li><code>LOCAL_IP</code>\uff1a The IP address of the node where the current container is located.</li> <li><code>NODE_IPS</code>\uff1a The IP addresses of the three nodes running the <code>OVN-IC</code> database, separated by commas.</li> </ul> <p>Next, deploy the follower of the <code>OVN-IC</code> DB on the other two nodes.</p> <p><code>docker</code> environment can use the following command.</p> <pre><code>docker run --name=ovn-ic-db -d --network=host --privileged -v /etc/ovn/:/etc/ovn -v /var/run/ovn:/var/run/ovn -v /var/log/ovn:/var/log/ovn -e LOCAL_IP=\"192.168.65.2\"  -e NODE_IPS=\"192.168.65.3,192.168.65.2,192.168.65.1\" -e LEADER_IP=\"192.168.65.3\"  kubeovn/kube-ovn:v1.13.0 bash start-ic-db.sh\n</code></pre> <p>If using <code>containerd</code> you can use the following command:</p> <pre><code>ctr -n k8s.io run -d --net-host --privileged --mount=\"type=bind,src=/etc/ovn/,dst=/etc/ovn,options=rbind:rw\" --mount=\"type=bind,src=/var/run/ovn,dst=/var/run/ovn,options=rbind:rw\" --mount=\"type=bind,src=/var/log/ovn,dst=/var/log/ovn,options=rbind:rw\"  --env=\"NODE_IPS=\"192.168.65.3,192.168.65.2,192.168.65.1\"\" --env=\"LOCAL_IP=\"192.168.65.2\"\" --env=\"LEADER_IP=\"192.168.65.3\"\" docker.io/kubeovn/kube-ovn:v1.13.0 ovn-ic-db bash start-ic-db.sh\n</code></pre> <ul> <li><code>LOCAL_IP</code>\uff1a The IP address of the node where the current container is located.</li> <li><code>NODE_IPS</code>\uff1a The IP addresses of the three nodes running the <code>OVN-IC</code> database, separated by commas.</li> <li><code>LEADER_IP</code>: The IP address of the <code>OVN-IC</code> DB leader node.</li> </ul> <p>Specify multiple <code>OVN-IC</code> database node addresses when creating <code>ovn-ic-config</code> for each cluster:</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: ovn-ic-config\nnamespace: kube-system\ndata:\nenable-ic: \"true\"\naz-name: \"az1\" ic-db-host: \"192.168.65.3,192.168.65.2,192.168.65.1\"\nic-nb-port: \"6645\"\nic-sb-port: \"6646\"\ngw-nodes: \"az1-gw\"\nauto-route: \"true\"\n</code></pre>"},{"location":"en/advance/with-ovn-ic/#manual-reset","title":"Manual Reset","text":"<p>In some cases, the entire interconnection configuration needs to be cleaned up due to configuration errors, you can refer to the following steps to clean up your environment.</p> <p>Delete the current <code>ovn-ic-config</code> Configmap:</p> <pre><code>kubectl -n kube-system delete cm ovn-ic-config\n</code></pre> <p>Delete <code>ts</code> logical switch:</p> <pre><code>kubectl-ko nbctl ls-del ts\n</code></pre> <p>Repeat the same steps at the peer cluster.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/with-submariner/","title":"Cluster Inter-Connection with Submariner","text":"<p>Submariner is an open source networking component that connects multiple Kubernetes cluster Pod and Service networks which can help Kube-OVN interconnect multiple clusters.</p> <p>Compared to OVN-IC, Submariner can connect Kube-OVN and non-Kube-OVN cluster networks, and Submariner can provide cross-cluster capability for services. However, Submariner currently only enables the default subnets to be connected, and cannot selectively connect multiple subnets.</p>"},{"location":"en/advance/with-submariner/#prerequisites","title":"Prerequisites","text":"<ul> <li>The Service CIDRs of the two clusters and the CIDR of the default Subnet cannot overlap.</li> </ul>"},{"location":"en/advance/with-submariner/#install-submariner","title":"Install Submariner","text":"<p>Download the <code>subctl</code> binary and deploy it to the appropriate path:</p> <pre><code>curl -Ls https://get.submariner.io | bash\nexport PATH=$PATH:~/.local/bin\necho export PATH=\\$PATH:~/.local/bin &gt;&gt; ~/.profile\n</code></pre> <p>Change <code>kubeconfig</code> context to the cluster that need to deploy <code>submariner-broker</code>:</p> <pre><code>subctl deploy-broker\n</code></pre> <p>In this document the default subnet CIDR for <code>cluster0</code> is <code>10.16.0.0/16</code> and the join subnet CIDR for <code>cluster0</code> is <code>100.64.0.0/16</code>, the default subnet CIDR for <code>cluster1</code> is <code>11.16.0.0/16</code> and the join subnet CIDR for <code>cluster1</code> is <code>100.68.0.0/16</code>.</p> <p>Switch <code>kubeconfig</code> to <code>cluster0</code> to register the cluster to the broker, and register the gateway node:</p> <pre><code>subctl  join broker-info.subm --clusterid  cluster0 --clustercidr 100.64.0.0/16,10.16.0.0/16  --natt=false --cable-driver vxlan --health-check=false\nkubectl label nodes cluster0 submariner.io/gateway=true\n</code></pre> <p>Switch <code>kubeconfig</code> to <code>cluster1</code> to register the cluster to the broker, and register the gateway node:</p> <pre><code>subctl  join broker-info.subm --clusterid  cluster1 --clustercidr 100.68.0.0/16,11.16.0.0/16  --natt=false --cable-driver vxlan --health-check=false\nkubectl label nodes cluster1 submariner.io/gateway=true\n</code></pre> <p>Next, you can start Pods in each of the two clusters and try to access each other using IPs.</p> <p>Network communication problems can be diagnosed by using the <code>subctl</code> command:</p> <pre><code>subctl show all\nsubctl diagnose all\n</code></pre> <p>For more Submariner operations please read Submariner Usage.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/custom-routes/","title":"Custom Routes","text":"<p>Custom routes can be configured via Pod's annotations. Here is an example:</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: custom-routes\nannotations:\novn.kubernetes.io/routes: |\n[{\n\"dst\": \"192.168.0.101/24\",\n\"gw\": \"10.16.0.254\"\n}, {\n\"gw\": \"10.16.0.254\"\n}]'\nspec:\ncontainers:\n- name: nginx\nimage: docker.io/library/nginx:alpine\n</code></pre> <p>Do not set the <code>dst</code> field if you want to configure the default route.</p> <p>For workloads such as Deployment, DaemonSet and StatefulSet, custom routes must be configured via <code>.spec.template.metadata.annotations</code>:</p> <pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nname: custom-routes\nlabels:\napp: nginx\nspec:\nreplicas: 2\nselector:\nmatchLabels:\napp: nginx\ntemplate:\nmetadata:\nlabels:\napp: nginx\nannotations:\novn.kubernetes.io/routes: |\n[{\n\"dst\": \"192.168.0.101/24\",\n\"gw\": \"10.16.0.254\"\n}, {\n\"gw\": \"10.16.0.254\"\n}]'\nspec:\ncontainers:\n- name: nginx\nimage: docker.io/library/nginx:alpine\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/dual-stack/","title":"DualStack","text":"<p>Different subnets in Kube-OVN can support different IP protocols. IPv4, IPv6 and dual-stack types of subnets can exist within one cluster. However, it is recommended to use a uniform protocol type within a cluster to simplify usage and maintenance.</p> <p>In order to support dual-stack, the host network needs to meet the dual-stack requirements, and the Kubernetes-related parameters need to be adjusted, please refer to official guide to dual-stack.</p>"},{"location":"en/guide/dual-stack/#create-dual-stack-subnet","title":"Create dual-stack Subnet","text":"<p>When configuring a dual stack Subnet, you only need to set the corresponding subnet CIDR format as <code>cidr=&lt;IPv4 CIDR&gt;,&lt;IPv6 CIDR&gt;</code>.</p> <p>The CIDR order requires IPv4 to come first and IPv6 to come second, as follows.</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata: name: ovn-test\nspec:\ncidrBlock: 10.16.0.0/16,fd00:10:16::/64\nexcludeIps:\n- 10.16.0.1\n- fd00:10:16::1\ngateway: 10.16.0.1,fd00:10:16::1\n</code></pre> <p>If you need to use a dual stack for the default subnet during installation, you need to change the following parameters in the installation script:</p> <pre><code>POD_CIDR=\"10.16.0.0/16,fd00:10:16::/64\"\nJOIN_CIDR=\"100.64.0.0/16,fd00:100:64::/64\"\n</code></pre>"},{"location":"en/guide/dual-stack/#check-pod-address","title":"Check Pod Address","text":"<p>Pods configured for dual-stack networks will be assigned both IPv4 and IPv6 addresses from that subnet, and the results will be displayed in the annotation of the Pod:</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nannotations:\novn.kubernetes.io/allocated: \"true\"\novn.kubernetes.io/cidr: 10.16.0.0/16,fd00:10:16::/64\novn.kubernetes.io/gateway: 10.16.0.1,fd00:10:16::1\novn.kubernetes.io/ip_address: 10.16.0.9,fd00:10:16::9\novn.kubernetes.io/logical_switch: ovn-default\novn.kubernetes.io/mac_address: 00:00:00:14:88:09\novn.kubernetes.io/network_types: geneve\novn.kubernetes.io/routed: \"true\"\n...\npodIP: 10.16.0.9\npodIPs:\n- ip: 10.16.0.9\n- ip: fd00:10:16::9\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/eip-snat/","title":"EIP and SNAT","text":"<p>This configuration is for the network under default VPC, for custom VPC please refer to VPC Gateway</p> <p>Kube-OVN supports SNAT and EIP functionality at the Pod level using the L3 Gateway feature in OVN. By using SNAT, a group of Pods can share an IP address for external access. With the EIP feature, a Pod can be directly associated with an external IP. External services can access the Pod directly through the EIP, and the Pod will also access external services through this EIP.</p> <p></p>"},{"location":"en/guide/eip-snat/#preparation","title":"Preparation","text":"<ul> <li>In order to use the OVN's L3 Gateway capability, a separate NIC must be bridged into the OVS bridge for overlay and underlay network communication.   The host must have other NICs for management.</li> <li>Since packets passing through NAT will go directly to the Underlay network, it is important to confirm that such packets can pass safely on the current network architecture.</li> <li>Currently, there is no conflict detection for EIP and SNAT addresses, and an administrator needs to manually assign them to avoid address conflicts.</li> </ul>"},{"location":"en/guide/eip-snat/#create-config","title":"Create Config","text":"<p>Create ConfigMap <code>ovn-external-gw-config</code> in <code>kube-system</code> Namespace:</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: ovn-external-gw-config\nnamespace: kube-system\ndata:\nenable-external-gw: \"true\"\nexternal-gw-nodes: \"kube-ovn-worker\"\nexternal-gw-nic: \"eth1\"\nexternal-gw-addr: \"172.56.0.1/16\"\nnic-ip: \"172.56.0.254/16\"\nnic-mac: \"16:52:f3:13:6a:25\"\n</code></pre> <ul> <li><code>enable-external-gw</code>: Whether to enable SNAT and EIP functions.</li> <li><code>type</code>: <code>centrailized</code> or <code>distributed</code>\uff0c Default is <code>centralized</code> If <code>distributed</code> is used, all nodes of the cluster need to have the same name NIC to perform the gateway function.</li> <li><code>external-gw-nodes</code>: In <code>centralized</code> mode\uff0cThe names of the node performing the gateway role, comma separated.</li> <li><code>external-gw-nic</code>: The name of the NIC that performs the role of a gateway on the node.</li> <li><code>external-gw-addr</code>: The IP and mask of the physical network gateway.</li> <li><code>nic-ip</code>,<code>nic-mac</code>: The IP and Mac assigned to the logical gateway port needs to be an unoccupied IP and Mac for the physical subnet.</li> </ul>"},{"location":"en/guide/eip-snat/#confirm-the-configuration-take-effect","title":"Confirm the Configuration Take Effect","text":"<p>Check the OVN-NB status to confirm that the <code>ovn-external</code> logical switch exists and that the correct address and chassis are bound to the <code>ovn-cluster-ovn-external</code> logical router port.</p> <pre><code># kubectl ko nbctl show\nswitch 3de4cea7-1a71-43f3-8b62-435a57ef16a6 (ovn-external)\nport ln-ovn-external\n        type: localnet\n        addresses: [\"unknown\"]\nport ovn-external-ovn-cluster\n        type: router\n        router-port: ovn-cluster-ovn-external\nrouter e1eb83ad-34be-4ed5-9a02-fcc8b1d357c4 (ovn-cluster)\nport ovn-cluster-ovn-external\n        mac: \"ac:1f:6b:2d:33:f1\"\nnetworks: [\"172.56.0.100/16\"]\ngateway chassis: [a5682814-2e2c-46dd-9c1c-6803ef0dab66]\n</code></pre> <p>Check the OVS status to confirm that the corresponding NIC is bridged into the <code>br-external</code> bridge:</p> <pre><code># kubectl ko vsctl ${gateway node name} show\ne7d81150-7743-4d6e-9e6f-5c688232e130\n    Bridge br-external\n        Port br-external\n            Interface br-external\n                type: internal\n        Port eno2\n            Interface eno2\n        Port patch-ln-ovn-external-to-br-int\n            Interface patch-ln-ovn-external-to-br-int\n                type: patch\n                options: {peer=patch-br-int-to-ln-ovn-external}\n</code></pre>"},{"location":"en/guide/eip-snat/#config-eip-amd-snat-on-pod","title":"Config EIP amd SNAT on Pod","text":"<p>SNAT and EIP can be configured by adding the <code>ovn.kubernetes.io/snat</code> or <code>ovn.kubernetes.io/eip</code> annotation to the Pod, respectively:</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: pod-gw\nannotations:\novn.kubernetes.io/snat: 172.56.0.200\nspec:\ncontainers:\n- name: snat-pod\nimage: docker.io/library/nginx:alpine\n---\napiVersion: v1\nkind: Pod\nmetadata:\nname: pod-gw\nannotations:\novn.kubernetes.io/eip: 172.56.0.233\nspec:\ncontainers:\n- name: eip-pod\nimage: docker.io/library/nginx:alpine\n</code></pre> <p>The EIP or SNAT rules configured by the Pod can be dynamically adjusted via kubectl or other tools, remember to remove the <code>ovn.kubernetes.io/routed</code> annotation to trigger the routing change.</p> <pre><code>kubectl annotate pod pod-gw ovn.kubernetes.io/eip=172.56.0.221 --overwrite\nkubectl annotate pod pod-gw ovn.kubernetes.io/routed-\n</code></pre> <p>When the EIP or SNAT takes into effect, the <code>ovn.kubernetes.io/routed</code> annotation will be added back.</p>"},{"location":"en/guide/eip-snat/#advanced-configuration","title":"Advanced Configuration","text":"<p>Some args of <code>kube-ovn-controller</code> allow for advanced configuration of SNAT and EIP:</p> <ul> <li><code>--external-gateway-config-ns</code>: The Namespace of Configmap <code>ovn-external-gw-config</code>, default is <code>kube-system</code>\u3002</li> <li><code>--external-gateway-net</code>: The name of the bridge to which the physical NIC is bridged, default is <code>external</code>.</li> <li><code>--external-gateway-vlanid</code>: Physical network Vlan Tag number, default is 0, i.e. no Vlan is used.</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/ippool/","title":"Configure IPPool","text":"<p>IPPool is a more granular IPAM management unit than Subnet. You can subdivide the subnet segment into multiple units through IPPool, and each unit is bound to one or more namespaces.</p>"},{"location":"en/guide/ippool/#instructions","title":"Instructions","text":"<p>Below is an example\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: IPPool\nmetadata:\nname: pool-1\nspec:\nsubnet: ovn-default\nips:\n- \"10.16.0.201\"\n- \"10.16.0.210/30\"\n- \"10.16.0.220..10.16.0.230\"\nnamespaces:\n- ns-1\n</code></pre> <p>Field description:</p> Field Usage Comment subnet Specify the subnet to which it belongs Required ips Specify IP ranges Support three formats: ,  and ... Support IPv6. namespaces Specifies the bound namespaces Optional"},{"location":"en/guide/ippool/#precautions","title":"Precautions","text":"<ol> <li>To ensure compatibility with Workload Universal IP Pool Fixed Address, the name of the IP pool cannot be an IP address;</li> <li>The <code>.spec.ips</code> of the IP pool can specify an IP address beyond the scope of the subnet, but the actual effective IP address is the intersection of <code>.spec.ips</code> and the CIDR of the subnet;</li> <li>Different IP pools of the same subnet cannot contain the same (effective) IP address;</li> <li>The <code>.spec.ips</code> of the IP pool can be modified dynamically;</li> <li>The IP pool will inherit the reserved IP of the subnet. When randomly assigning an IP address from the IP pool, the reserved IP included in the IP pool will be skipped;</li> <li>When randomly assigning an IP address from a subnet, it will only be assigned from a range other than all IP pools in the subnet.</li> </ol> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/loadbalancer-service/","title":"LoadBalancer Type Service","text":"<p>Kube-OVN supports the implementation of VPC and VPC gateway. For specific configurations, please refer to the VPC configuration.</p> <p>Due to the complexity of using VPC gateways, the implementation based on VPC gateways has been simplified. It supports creating LoadBalancer type Services in the default VPC, allowing access to Services in the default VPC through LoadBalancerIP.</p> <p>First, make sure the following conditions are met in the environment:</p> <ol> <li>Install <code>multus-cni</code> and <code>macvlan cni</code>\u3002</li> <li>LoadBalancer Service support relies on simplified implementation of VPC gateway code, still utilizing the vpc-nat-gw image and depending on macvlan for multi-interface functionality support.</li> <li>Currently, it only supports configuration in the default VPC. Support for LoadBalancers in custom VPCs can be referred to in the VPC configuration.</li> </ol>"},{"location":"en/guide/loadbalancer-service/#steps-to-configure-default-vpc-loadbalancer-service","title":"Steps to Configure Default VPC LoadBalancer Service","text":""},{"location":"en/guide/loadbalancer-service/#enable-feature-flag","title":"Enable Feature Flag","text":"<p>Modify the deployment <code>kube-ovn-controller</code> under the kube-system namespace and add the parameter <code>--enable-lb-svc=true</code> to the <code>args</code> section to enable the feature (by default it's set to false).</p> <pre><code>containers:\n- args:\n- /kube-ovn/start-controller.sh\n- --default-cidr=10.16.0.0/16\n- --default-gateway=10.16.0.1\n- --default-gateway-check=true\n- --enable-lb-svc=true                  // parameter is set to true\n</code></pre>"},{"location":"en/guide/loadbalancer-service/#create-networkattachmentdefinition-crd-resource","title":"Create NetworkAttachmentDefinition CRD Resource","text":"<p>Refer to the following YAML and create the <code>net-attach-def</code> resource:</p> <pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: lb-svc-attachment\nnamespace: kube-system\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.0\",\n\"type\": \"macvlan\",\n\"master\": \"eth0\",                            //Physical network card, configure according to the actual situation\n\"mode\": \"bridge\"\n}'\n</code></pre> <p>By default, the physical NIC <code>eth0</code> is used to implement the multi-interface functionality. If another physical NIC is needed, modify the <code>master</code> value to specify the name of the desired physical NIC.</p>"},{"location":"en/guide/loadbalancer-service/#create-subnet","title":"Create Subnet","text":"<p>The created Subnet is used to allocate LoadBalancerIP for the LoadBalancer Service, which should normally  be accessible from outside the cluster. An Underlay Subnet can be configured for address allocation.</p> <p>Refer to the following YAML to create a new subnet:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: attach-subnet\nspec:\nprotocol: IPv4\nprovider: lb-svc-attachment.kube-system          //The provider format is fixed and consists of the Name.Namespace of the net-attach-def resource created in the previous step\ncidrBlock: 172.18.0.0/16\ngateway: 172.18.0.1\nexcludeIps:\n- 172.18.0.0..172.18.0.10\n</code></pre> <p>In the <code>provider</code> parameter of the Subnet, <code>ovn</code> or <code>.ovn</code> suffix is used to indicate that the subnet is managed by Kube-OVN and requires corresponding logical switch records to be created.</p> <p>If <code>provider</code> is neither <code>ovn</code> nor ends with <code>.ovn</code>, Kube-OVN only provides the IPAM functionality to record IP address allocation without handling business logic for the subnet.</p>"},{"location":"en/guide/loadbalancer-service/#create-loadbalancer-service","title":"Create LoadBalancer Service","text":"<p>Refer to the following YAML to create a LoadBalancer Service:</p> <pre><code>apiVersion: v1\nkind: Service\nmetadata:\nannotations:\nlb-svc-attachment.kube-system.kubernetes.io/logical_switch: attach-subnet #Optional\novn.kubernetes.io/attachmentprovider: lb-svc-attachment.kube-system #Required\nlabels:\napp: dynamic\nname: test-service\nnamespace: default\nspec:\nloadBalancerIP: 172.18.0.18 #Optional\nports:\n- name: test\nprotocol: TCP\nport: 80\ntargetPort: 80\nselector:\napp: dynamic\nsessionAffinity: None\ntype: LoadBalancer\n</code></pre> <p>In the yaml, the annotation <code>ovn.kubernetes.io/attachmentprovider</code> is required, and its value is composed of the Name.Namespace of the <code>net-attach-def</code> resource created in the first step. This annotation is used to find the <code>net-attach-def</code> resources when creating Pods.</p> <p>The subnet used for multi-interface address allocation can be specified through an annotation. The annotation key format is <code>net-attach-def</code> resource's <code>Name.Namespace.kubernetes.io/logical_switch</code>. This configuration is <code>optional</code> and if LoadBalancerIP address is not specified, addresses will be dynamically allocated from this subnet and filled into the LoadBalancerIP field.</p> <p>If a static LoadBalancerIP address is required, the <code>spec.loadBalancerIP</code> field can be configured. The address must be within the specified subnet's address range.</p> <p>After creating the Service using the YAML, you can see the Pod startup information in the same namespace as the Service:</p> <pre><code># kubectl get pod\nNAME READY STATUS RESTARTS AGE\nlb-svc-test-service-6869d98dd8-cjvll 1/1 Running 0 107m\n# kubectl get svc\nNAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\ntest-service LoadBalancer 10.109.201.193 172.18.0.18 80:30056/TCP 107m\n</code></pre> <p>When specifying the <code>service.spec.loadBalancerIP</code> parameter, it will be assigned to the service's external IP field. If not specified, the parameter will be assigned a random value.</p> <p>View the YAML output of the test Pod to see the assigned multi-interface addresses:</p> <pre><code># kubectl get pod -o yaml lb-svc-test-service-6869d98dd8-cjvll\napiVersion: v1\nkind: Pod\nmetadata:\n   annotations:\n     k8s.v1.cni.cncf.io/network-status: |-\n       [{\n\"name\": \"kube-ovn\",\n           \"ips\": [\n\"10.16.0.2\"\n],\n           \"default\": true,\n           \"dns\": {}\n},{\n\"name\": \"default/test-service\",\n           \"interface\": \"net1\",\n           \"mac\": \"ba:85:f7:02:9f:42\",\n           \"dns\": {}\n}]\nk8s.v1.cni.cncf.io/networks: default/test-service\n     k8s.v1.cni.cncf.io/networks-status: |-\n       [{\n\"name\": \"kube-ovn\",\n           \"ips\": [\n\"10.16.0.2\"\n],\n           \"default\": true,\n           \"dns\": {}\n},{\n\"name\": \"default/test-service\",\n           \"interface\": \"net1\",\n           \"mac\": \"ba:85:f7:02:9f:42\",\n           \"dns\": {}\n}]\novn.kubernetes.io/allocated: \"true\"\novn.kubernetes.io/cidr: 10.16.0.0/16\n     ovn.kubernetes.io/gateway: 10.16.0.1\n     ovn.kubernetes.io/ip_address: 10.16.0.2\n     ovn.kubernetes.io/logical_router: ovn-cluster\n     ovn.kubernetes.io/logical_switch: ovn-default\n     ovn.kubernetes.io/mac_address: 00:00:00:45:F4:29\n     ovn.kubernetes.io/pod_nic_type: veth-pair\n     ovn.kubernetes.io/routed: \"true\"\ntest-service.default.kubernetes.io/allocated: \"true\"\ntest-service.default.kubernetes.io/cidr: 172.18.0.0/16\n     test-service.default.kubernetes.io/gateway: 172.18.0.1\n     test-service.default.kubernetes.io/ip_address: 172.18.0.18\n     test-service.default.kubernetes.io/logical_switch: attach-subnet\n     test-service.default.kubernetes.io/mac_address: 00:00:00:AF:AA:BF\n     test-service.default.kubernetes.io/pod_nic_type: veth-pair\n</code></pre> <p>Check the service information:</p> <pre><code># kubectl get svc -o yaml test-service\napiVersion: v1\nkind: Service\nmetadata:\n   annotations:\n     kubectl.kubernetes.io/last-applied-configuration: |\n{\"apiVersion\":\"v1\",\"kind\":\"Service\",\"metadata\":{\"annotations\":{\"test-service.default.kubernetes.io/logical_switch\":\"attach-subnet\"},\"labels \":{\"app\":\"dynamic\"},\"name\":\"test-service\",\"namespace\":\"default\"},\"spec\":{\"ports\":[{\"name\":\"test\", \"port\":80,\"protocol\":\"TCP\",\"targetPort\":80}],\"selector\":{\"app\":\"dynamic\"},\"sessionAffinity\":\"None\",\"type\":\"LoadBalancer \"}}\novn.kubernetes.io/vpc:ovn-cluster\n     test-service.default.kubernetes.io/logical_switch: attach-subnet\n   creationTimestamp: \"2022-06-15T09:01:58Z\"\nlabels:\n     app: dynamic\n   name: test-service\n   namespace: default\n   resourceVersion: \"38485\"\nuid: 161edee1-7f6e-40f5-9e09-5a52c44267d0\nspec:\n   allocateLoadBalancerNodePorts: true\nclusterIP: 10.109.201.193\n   clusterIPs:\n   - 10.109.201.193\n   externalTrafficPolicy: Cluster\n   internalTrafficPolicy: Cluster\n   ipFamilies:\n   - IPv4\n   ipFamilyPolicy: SingleStack\n   ports:\n   - name: test\nnodePort: 30056\nport: 80\nprotocol: TCP\n     targetPort: 80\nselector:\n     app: dynamic\n   sessionAffinity: None\n   type: LoadBalancer\nstatus:\n   loadBalancer:\n     ingress:\n     - ip: 172.18.0.18\n</code></pre>"},{"location":"en/guide/loadbalancer-service/#testing-loadbalancerip-access","title":"Testing LoadBalancerIP access","text":"<p>Refer to the following YAML to create a test Pod that serves as the Endpoints for the Service:</p> <pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nlabels:\napp: dynamic\nname: dynamic\nnamespace: default\nspec:\nreplicas: 2\nselector:\nmatchLabels:\napp: dynamic\nstrategy:\nrollingUpdate:\nmaxSurge: 25%\nmaxUnavailable: 25%\ntype: RollingUpdate\ntemplate:\nmetadata:\ncreationTimestamp: null\nlabels:\napp: dynamic\nspec:\ncontainers:\n- image: docker.io/library/nginx:alpine\nimagePullPolicy: IfNotPresent\nname: nginx\ndnsPolicy: ClusterFirst\nrestartPolicy: Always\n</code></pre> <p>Under normal circumstances, the provided subnet addresses should be accessible from outside the cluster. To verify, access the Service's <code>LoadBalancerIP:Port</code> from within the cluster and check if the access is successful.</p> <pre><code># curl 172.18.0.11:80\n&lt;html&gt;\n&lt;head&gt;\n        &lt;title&gt;Hello World!&lt;/title&gt;\n        &lt;link href='//fonts.googleapis.com/css?family=Open+Sans:400,700' rel='stylesheet' type='text/css'&gt;\n        &lt;style&gt;\n        body {\nbackground-color: white;\ntext-align: center;\npadding: 50px;\nfont-family: \"Open Sans\",\"Helvetica Neue\",Helvetica,Arial,sans-serif;\n}\n#logo {\nmargin-bottom: 40px;\n}\n&lt;/style&gt;\n&lt;/head&gt;\n&lt;body&gt;\n                &lt;h1&gt;Hello World!&lt;/h1&gt;\n                                &lt;h3&gt;Links found&lt;/h3&gt;\n        &lt;h3&gt;I am on  dynamic-7d8d7874f5-hsgc4&lt;/h3&gt;\n        &lt;h3&gt;Cookie                  =&lt;/h3&gt;\n                                        &lt;b&gt;KUBERNETES&lt;/b&gt; listening in 443 available at tcp://10.96.0.1:443&lt;br /&gt;\n                                                &lt;h3&gt;my name is hanhouchao!&lt;/h3&gt;\n                        &lt;h3&gt; RequestURI='/'&lt;/h3&gt;\n&lt;/body&gt;\n&lt;/html&gt;\n</code></pre> <p>Enter the Pod created by the Service and check the network information:</p> <pre><code># ip a\n4: net1@if62: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1500 qdisc noqueue state UP group default\n    link/ether ba:85:f7:02:9f:42 brd ff:ff:ff:ff:ff:ff link-netnsid 0\ninet 172.18.0.18/16 scope global net1\n       valid_lft forever preferred_lft forever\n    inet6 fe80::b885:f7ff:fe02:9f42/64 scope link\n       valid_lft forever preferred_lft forever\n36: eth0@if37: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1400 qdisc noqueue state UP group default\n    link/ether 00:00:00:45:f4:29 brd ff:ff:ff:ff:ff:ff link-netnsid 0\ninet 10.16.0.2/16 brd 10.16.255.255 scope global eth0\n       valid_lft forever preferred_lft forever\n    inet6 fe80::200:ff:fe45:f429/64 scope link\n       valid_lft forever preferred_lft forever\n\n# ip rule\n0: from all lookup local\n32764: from all iif eth0 lookup 100\n32765: from all iif net1 lookup 100\n32766: from all lookup main\n32767: from all lookup default\n\n# ip route show table 100\ndefault via 172.18.0.1 dev net1\n10.109.201.193 via 10.16.0.1 dev eth0\n172.18.0.0/16 dev net1 scope link\n\n# iptables -t nat -L -n -v\nChain PREROUTING (policy ACCEPT 0 packets, 0 bytes)\npkts bytes target     prot opt in     out     source               destination\n    0     0 DNAT       tcp  --  *      *       0.0.0.0/0            172.18.0.18          tcp dpt:80 to:10.109.201.193:80\n\nChain INPUT (policy ACCEPT 0 packets, 0 bytes)\npkts bytes target     prot opt in     out     source               destination\n\nChain OUTPUT (policy ACCEPT 0 packets, 0 bytes)\npkts bytes target     prot opt in     out     source               destination\n\nChain POSTROUTING (policy ACCEPT 0 packets, 0 bytes)\npkts bytes target     prot opt in     out     source               destination\n    0     0 MASQUERADE  all  --  *      *       0.0.0.0/0            10.109.201.193\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/mirror/","title":"Traffic Mirror","text":"<p>The traffic mirroring feature allows packets to and from the container network to be copied to a specific NIC of the host. Administrators or developers can listen to this NIC to get the complete container network traffic for further analysis, monitoring, security auditing and other operations. It can also be integrated with traditional NPM for more fine-grained traffic visibility.</p> <p>The traffic mirroring feature introduces some performance loss, with an additional CPU consumption of 5% to 10% depending on CPU performance and traffic characteristics.</p> <p></p>"},{"location":"en/guide/mirror/#global-traffic-mirroring-settings","title":"Global Traffic Mirroring Settings","text":"<p>The traffic mirroring is disabled by default, please modify the args of <code>kube-ovn-cni</code> DaemonSet to enable it:</p> <ul> <li><code>--enable-mirror=true</code>: Whether to enable traffic mirroring.</li> <li><code>--mirror-iface=mirror0</code>: The name of the NIC that the traffic mirror is copied to. This NIC can be a physical NIC that already exists on the host machine.   At this point the NIC will be bridged into the br-int bridge and the mirrored traffic will go directly to the underlying switch.   If the NIC name does not exist, Kube-OVN will automatically create a virtual NIC with the same name, through which the administrator or developer can access all traffic on the current node on the host.   The default is <code>mirror0</code>.</li> </ul> <p>Next, you can listen to the traffic on <code>mirror0</code> with tcpdump or other traffic analysis tools.</p> <pre><code>tcpdump -ni mirror0\n</code></pre>"},{"location":"en/guide/mirror/#pod-level-mirroring-settings","title":"Pod Level Mirroring Settings","text":"<p>If you only need to mirror some Pod traffic, you need to disable the global traffic mirroring and then add the <code>ovn.kubernetes.io/mirror</code> annotation on a specific Pod to enable Pod-level traffic mirroring.</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: mirror-pod\nnamespace: ls1\nannotations:\novn.kubernetes.io/mirror: \"true\"\nspec:\ncontainers:\n- name: mirror-pod\nimage: docker.io/library/nginx:alpine\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/networkpolicy-log/","title":"NetworkPolicy Logging","text":"<p>NetworkPolicy is a interface provided by Kubernetes and implemented by Kube-OVN through OVN's ACLs. With NetworkPolicy, if the networks are down, it is difficult to determine whether it is caused by a network failure or a NetworkPolicy rule problem. Kube-OVN provides NetworkPolicy logging to help administrators quickly locate whether a NetworkPolicy drop rule has been hit, and to record the illegal accesses.</p> <p>Once NetworkPolicy logging is turned on, logs need to be printed for every packet that hits a Drop rule, which introduces additional performance overhead. Under a malicious attack, a large number of logs in a short period of time may exhaust the CPU. We recommend turning off logging by default in production environments and dynamically turning it on when you need to troubleshoot problems.</p>"},{"location":"en/guide/networkpolicy-log/#enable-networkpolicy-logging","title":"Enable NetworkPolicy Logging","text":"<p>Add the annotation <code>ovn.kubernetes.io/enable_log</code> to the NetworkPolicy where logging needs to be enabled, as follows:</p> <pre><code>apiVersion: networking.k8s.io/v1\nkind: NetworkPolicy\nmetadata:\nname: default-deny-ingress\nnamespace: kube-system\nannotations:\novn.kubernetes.io/enable_log: \"true\"\nspec:\npodSelector: {}\npolicyTypes:\n- Ingress\n</code></pre> <p>Next, you can observe the log of dropped packets in <code>/var/log/ovn/ovn-controller.log</code> on the host of the corresponding Pod:</p> <pre><code># tail -f /var/log/ovn/ovn-controller.log\n2022-07-20T05:55:03.229Z|00394|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: udp,vlan_tci=0x0000,dl_src=00:00:00:21:b7:d1,dl_dst=00:00:00:8d:0b:86,nw_src=10.16.0.10,nw_dst=10.16.0.7,nw_tos=0,nw_ecn=0,nw_ttl=63,tp_src=54343,tp_dst=53\n2022-07-20T05:55:06.229Z|00395|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: udp,vlan_tci=0x0000,dl_src=00:00:00:21:b7:d1,dl_dst=00:00:00:8d:0b:86,nw_src=10.16.0.9,nw_dst=10.16.0.7,nw_tos=0,nw_ecn=0,nw_ttl=63,tp_src=44187,tp_dst=53\n2022-07-20T05:55:08.230Z|00396|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: udp,vlan_tci=0x0000,dl_src=00:00:00:21:b7:d1,dl_dst=00:00:00:8d:0b:86,nw_src=10.16.0.10,nw_dst=10.16.0.7,nw_tos=0,nw_ecn=0,nw_ttl=63,tp_src=54274,tp_dst=53\n2022-07-20T05:55:11.231Z|00397|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: udp,vlan_tci=0x0000,dl_src=00:00:00:21:b7:d1,dl_dst=00:00:00:8d:0b:86,nw_src=10.16.0.9,nw_dst=10.16.0.7,nw_tos=0,nw_ecn=0,nw_ttl=63,tp_src=32778,tp_dst=53\n2022-07-20T05:55:11.231Z|00398|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: udp,vlan_tci=0x0000,dl_src=00:00:00:21:b7:d1,dl_dst=00:00:00:8d:0b:86,nw_src=10.16.0.9,nw_dst=10.16.0.7,nw_tos=0,nw_ecn=0,nw_ttl=63,tp_src=34188,tp_dst=53\n2022-07-20T05:55:13.231Z|00399|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: udp,vlan_tci=0x0000,dl_src=00:00:00:21:b7:d1,dl_dst=00:00:00:8d:0b:86,nw_src=10.16.0.10,nw_dst=10.16.0.7,nw_tos=0,nw_ecn=0,nw_ttl=63,tp_src=43290,tp_dst=53\n2022-07-20T05:55:22.096Z|00400|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: icmp,vlan_tci=0x0000,dl_src=00:00:00:6c:42:91,dl_dst=00:00:00:a5:d7:63,nw_src=10.16.0.9,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0\n2022-07-20T05:55:22.097Z|00401|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: icmp,vlan_tci=0x0000,dl_src=00:00:00:6c:42:91,dl_dst=00:00:00:a5:d7:63,nw_src=10.16.0.9,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0\n2022-07-20T05:55:22.098Z|00402|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: icmp,vlan_tci=0x0000,dl_src=00:00:00:6c:42:91,dl_dst=00:00:00:a5:d7:63,nw_src=10.16.0.9,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0\n</code></pre>"},{"location":"en/guide/networkpolicy-log/#disable-networkpolicy-logging","title":"Disable NetworkPolicy Logging","text":"<p>Set annotation <code>ovn.kubernetes.io/enable_log</code> in the corresponding NetworkPolicy to <code>false</code> to disable NetworkPolicy logging:</p> <pre><code>kubectl annotate networkpolicy -n kube-system default-deny-ingress ovn.kubernetes.io/enable_log=false --overwrite\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/prometheus-grafana/","title":"Monitor and Dashboard","text":"<p>Kube-OVN can export network control plane information and network data plane quality information metrics to the external in formats supported by Prometheus.</p> <p>We use the CRD provided by kube-prometheus to define the corresponding Prometheus monitoring rules. For all monitoring metrics supported by Kube-OVN, please refer to Kube-OVN Monitoring Metrics.</p> <p>If you are using native Prometheus, please refer to Configuring Native Prometheus for configuration.</p>"},{"location":"en/guide/prometheus-grafana/#install-prometheus-monitor","title":"Install Prometheus Monitor","text":"<p>Kube-OVN uses Prometheus Monitor CRD to manage the monitoring output.</p> <pre><code># network quality related monitoring metrics\nkubectl apply -f https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/pinger-monitor.yaml\n# kube-ovn-controller metrics\nkubectl apply -f https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/controller-monitor.yaml\n# kube-ovn-cni metrics\nkubectl apply -f https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/cni-monitor.yaml\n# ovn metrics\nkubectl apply -f https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/ovn-monitor.yaml\n</code></pre> <p>The default interval for Prometheus pull is 15s, if you need to adjust it, modify the <code>interval</code> value in yaml.</p>"},{"location":"en/guide/prometheus-grafana/#import-grafana-dashboard","title":"Import Grafana Dashboard","text":"<p>Kube-OVN provides a predefined Grafana Dashboard to display control plane and data plane related metrics.</p> <p>Download the corresponding Dashboard template:</p> <pre><code># network quality related monitoring dashboard\nwget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/pinger-grafana.json\n# kube-ovn-controller dashboard\nwget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/controller-grafana.json\n# kube-ovn-cni dashboard\nwget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/cni-grafana.json\n# ovn dashboard\nwget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/ovn-grafana.json\n# ovs dashboard\nwget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/ovs-grafana.json\n</code></pre> <p>Import these templates into Grafana and set the data source to the corresponding Prometheus to see the following Dashboards.</p> <p><code>kube-ovn-controller</code> dashboard:</p> <p></p> <p><code>kube-ovn-pinger</code> dashboard:</p> <p></p> <p><code>kube-ovn-cni</code> dashboard:</p> <p></p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/prometheus/","title":"Config Native Prometheus","text":"<p>Kube-OVN provides rich monitoring data for OVN/OVS health status checks and connectivity checks of container and host networks, and Kube-OVN is configured with ServiceMonitor for Prometheus to dynamically obtain monitoring metrics.</p> <p>In some cases, where only Prometheus Server is installed and no other components are installed, you can dynamically obtain monitoring data for the cluster environment by modifying the configuration of Prometheus.</p>"},{"location":"en/guide/prometheus/#config-prometheus","title":"Config Prometheus","text":"<p>The following configuration documentation, referenced from Prometheus Service Discovery.</p>"},{"location":"en/guide/prometheus/#permission-configuration","title":"Permission Configuration","text":"<p>Prometheus is deployed in the cluster and needs to access the k8s apiserver to query the monitoring data of the containers.</p> <p>Refer to the following yaml to configure the permissions required by Prometheus:</p> <pre><code>apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\nname: prometheus\nrules:\n- apiGroups: [\"\"]\nresources:\n- nodes\n- nodes/proxy\n- services\n- endpoints\n- pods\nverbs: [\"get\", \"list\", \"watch\"]\n- apiGroups:\n- extensions\nresources:\n- ingresses\nverbs: [\"get\", \"list\", \"watch\"]\n- nonResourceURLs: [\"/metrics\"]\nverbs: [\"get\"]\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\nname: prometheus\nnamespace: default\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\nname: prometheus\nroleRef:\napiGroup: rbac.authorization.k8s.io\nkind: ClusterRole\nname: prometheus\nsubjects:\n- kind: ServiceAccount\nname: prometheus\nnamespace: default\n</code></pre>"},{"location":"en/guide/prometheus/#prometheus-configmap","title":"Prometheus ConfigMap","text":"<p>The startup of Prometheus relies on the configuration file prometheus.yml, the contents of which can be configured in ConfigMap and dynamically mounted to the Pod.</p> <p>Create the ConfigMap file used by Prometheus by referring to the following yaml:</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: prometheus-config\ndata:\nprometheus.yml: |-\nglobal:\nscrape_interval:     15s \nevaluation_interval: 15s\nscrape_configs:\n- job_name: 'prometheus'\nstatic_configs:\n- targets: ['localhost:9090']\n\n- job_name: 'kubernetes-nodes'\ntls_config:\nca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\nbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\nkubernetes_sd_configs:\n- role: node\n\n- job_name: 'kubernetes-service'\ntls_config:\nca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\nbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\nkubernetes_sd_configs:\n- role: service\n\n- job_name: 'kubernetes-endpoints'\ntls_config:\nca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\nbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\nkubernetes_sd_configs:\n- role: endpoints\n\n- job_name: 'kubernetes-ingress'\ntls_config:\nca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\nbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\nkubernetes_sd_configs:\n- role: ingress\n\n- job_name: 'kubernetes-pods'\ntls_config:\nca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\nbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\nkubernetes_sd_configs:\n- role: pod\n</code></pre> <p>Prometheus provides role-based querying of Kubernetes resource monitoring operations, which can be configured in the official documentation kubernetes_sd_config\u3002</p>"},{"location":"en/guide/prometheus/#deploy-prometheus","title":"Deploy Prometheus","text":"<p>Deploy Prometheus Server by referring to the following yaml:</p> <pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nlabels:\napp: prometheus\nname: prometheus\nnamespace: default\nspec:\nreplicas: 1\nselector:\nmatchLabels:\napp: prometheus\nstrategy:\nrollingUpdate:\nmaxSurge: 25%\nmaxUnavailable: 25%\ntype: RollingUpdate\ntemplate:\nmetadata:\nlabels:\napp: prometheus\nspec:\nserviceAccountName: prometheus\nserviceAccount: prometheus\ncontainers:\n- image: docker.io/prom/prometheus:latest\nimagePullPolicy: IfNotPresent\nname: prometheus\ncommand:\n- \"/bin/prometheus\"\nargs:\n- \"--config.file=/etc/prometheus/prometheus.yml\"\nports:\n- containerPort: 9090\nprotocol: TCP\nvolumeMounts:\n- mountPath: \"/etc/prometheus\"\nname: prometheus-config\nvolumes:\n- name: prometheus-config\nconfigMap:\nname: prometheus-config\n</code></pre> <p>Deploy Prometheus Service by referring to the following yaml:</p> <pre><code>kind: Service\napiVersion: v1\nmetadata:\nname: prometheus\nnamespace: default\nlabels:\nname: prometheus\nspec:\nports:\n- name: test\nprotocol: TCP\nport: 9090\ntargetPort: 9090\ntype: NodePort\nselector:\napp: prometheus\nsessionAffinity: None\n</code></pre> <p>After exposing Prometheus through NodePort, Prometheus can be accessed through the node address.</p>"},{"location":"en/guide/prometheus/#prometheus-metrics-config","title":"Prometheus Metrics Config","text":"<p>View information about Prometheus on the environment:</p> <pre><code># kubectl get svc \nNAME         TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)          AGE\nkubernetes   ClusterIP   10.4.0.1       &lt;none&gt;        443/TCP          8d\nprometheus   NodePort    10.4.102.222   &lt;none&gt;        9090:32611/TCP   8d\n# kubectl get pod -o wide\nNAME                          READY   STATUS    RESTARTS   AGE    IP          NODE              NOMINATED NODE   READINESS GATES\nprometheus-7544b6b84d-v9m8s   1/1     Running   0          3d5h   10.3.0.7    192.168.137.219   &lt;none&gt;           &lt;none&gt;\n# kubectl get endpoints -o wide\nNAME         ENDPOINTS                                                        AGE\nkubernetes   192.168.136.228:6443,192.168.136.232:6443,192.168.137.219:6443   8d\nprometheus   10.3.0.7:9090                                                    8d\n</code></pre> <p>Access Prometheus via NodePort to see the data dynamically queried by Status/Service Discovery:</p> <p></p> <p>You can see that you can currently query all the service data information on the cluster.</p>"},{"location":"en/guide/prometheus/#configure-to-query-specified-resource","title":"Configure to Query Specified Resource","text":"<p>The ConfigMap configuration above queries all resource data. If you only need resource data for a certain role, you can add filter conditions.</p> <p>Take Service as an example, modify the ConfigMap content to query only the service monitoring data:</p> <pre><code>    - job_name: 'kubernetes-service'\ntls_config:\nca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\nbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\nkubernetes_sd_configs:\n- role: service\nrelabel_configs:\n- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]\naction: \"keep\"\nregex: \"true\"\n- action: labelmap\nregex: __meta_kubernetes_service_label_(.+)\n- source_labels: [__meta_kubernetes_namespace]\ntarget_label: kubernetes_namespace\n- source_labels: [__meta_kubernetes_service_name]\ntarget_label: kubernetes_service_name\n- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]\naction: replace\ntarget_label: __metrics_path__\nregex: \"(.+)\"\n</code></pre> <p>Check the Kube-OVN Service in kube-system Namespace:</p> <pre><code># kubectl get svc -n kube-system\nNAME                  TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)                  AGE\nkube-dns              ClusterIP   10.4.0.10      &lt;none&gt;        53/UDP,53/TCP,9153/TCP   13d\nkube-ovn-cni          ClusterIP   10.4.228.60    &lt;none&gt;        10665/TCP                13d\nkube-ovn-controller   ClusterIP   10.4.172.213   &lt;none&gt;        10660/TCP                13d\nkube-ovn-monitor      ClusterIP   10.4.242.9     &lt;none&gt;        10661/TCP                13d\nkube-ovn-pinger       ClusterIP   10.4.122.52    &lt;none&gt;        8080/TCP                 13d\novn-nb                ClusterIP   10.4.80.213    &lt;none&gt;        6641/TCP                 13d\novn-northd            ClusterIP   10.4.126.234   &lt;none&gt;        6643/TCP                 13d\novn-sb                ClusterIP   10.4.216.249   &lt;none&gt;        6642/TCP                 13d\n</code></pre> <p>Add annotation <code>prometheus.io/scrape=\"true\"</code> to Service\uff1a</p> <pre><code># kubectl annotate svc -n kube-system kube-ovn-cni  prometheus.io/scrape=true\nservice/kube-ovn-cni annotated\n# kubectl annotate svc -n kube-system kube-ovn-controller  prometheus.io/scrape=true\nservice/kube-ovn-controller annotated\n# kubectl annotate svc -n kube-system kube-ovn-monitor  prometheus.io/scrape=true\nservice/kube-ovn-monitor annotated\n# kubectl annotate svc -n kube-system kube-ovn-pinger  prometheus.io/scrape=true\nservice/kube-ovn-pinger annotated\n</code></pre> <p>Check the configured Service information:</p> <pre><code># kubectl get svc -o yaml -n kube-system kube-ovn-controller\napiVersion: v1\nkind: Service\nmetadata:\n  annotations:\n    helm.sh/chart-version: v3.10.0-alpha.55\n    helm.sh/original-name: kube-ovn-controller\n    ovn.kubernetes.io/vpc: ovn-cluster\n    prometheus.io/scrape: \"true\"                        // added annotation\n  labels:\n    app: kube-ovn-controller\n  name: kube-ovn-controller\n  namespace: kube-system\nspec:\n  clusterIP: 10.4.172.213\n  clusterIPs:\n  - 10.4.172.213\n  internalTrafficPolicy: Cluster\n  ipFamilies:\n  - IPv4\n  ipFamilyPolicy: SingleStack\n  ports:\n  - name: metrics\n    port: 10660\nprotocol: TCP\n    targetPort: 10660\nselector:\n    app: kube-ovn-controller\n  sessionAffinity: None\n  type: ClusterIP\nstatus:\n  loadBalancer: {}\n</code></pre> <p>Looking at the Prometheus Status Targets information, you can only see the Services with annotation:</p> <p></p> <p>For more information about adding filter parameters to relabel, please check Prometheus-Relabel\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/qos/","title":"Manage QoS","text":"<p>Kube-OVN supports two types of Pod level QoS:</p> <ul> <li>Maximum bandwidth limit QoS.</li> <li><code>linux-netem</code>, QoS for simulating latency and packet loss that can be used for simulation testing.</li> </ul> <p>Currently, only Pod level QoS is supported, and QoS restrictions at the Namespace or Subnet level are not supported.</p>"},{"location":"en/guide/qos/#maximum-bandwidth-limit-qos","title":"Maximum Bandwidth Limit QoS","text":"<p>This type of QoS can be dynamically configured via Pod annotation and can be adjusted without restarting running Pod. Bandwidth speed limit unit is <code>Mbit/s</code>.</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: qos\nnamespace: ls1\nannotations:\novn.kubernetes.io/ingress_rate: \"3\"\novn.kubernetes.io/egress_rate: \"1\"\nspec:\ncontainers:\n- name: qos\nimage: docker.io/library/nginx:alpine\n</code></pre> <p>Use annotation to dynamically adjust QoS:</p> <pre><code>kubectl annotate --overwrite  pod nginx-74d5899f46-d7qkn ovn.kubernetes.io/ingress_rate=3\n</code></pre>"},{"location":"en/guide/qos/#test-qos","title":"Test QoS","text":"<p>Deploy the containers needed for performance testing:</p> <pre><code>kind: DaemonSet\napiVersion: apps/v1\nmetadata:\nname: perf\nnamespace: ls1\nlabels:\napp: perf\nspec:\nselector:\nmatchLabels:\napp: perf\ntemplate:\nmetadata:\nlabels:\napp: perf\nspec:\ncontainers:\n- name: nginx\nimage: docker.io/kubeovn/perf\n</code></pre> <p>Exec into one Pod and run iperf3 server:</p> <pre><code># kubectl exec -it perf-4n4gt -n ls1 sh\n# iperf3 -s\n-----------------------------------------------------------\nServer listening on 5201\n-----------------------------------------------------------\n</code></pre> <p>Exec into the other Pod and run iperf3 client to connect above server address:</p> <pre><code># kubectl exec -it perf-d4mqc -n ls1 sh\n# iperf3 -c 10.66.0.12\nConnecting to host 10.66.0.12, port 5201\n[  4] local 10.66.0.14 port 51544 connected to 10.66.0.12 port 5201\n[ ID] Interval           Transfer     Bandwidth       Retr  Cwnd\n[  4]   0.00-1.00   sec  86.4 MBytes   725 Mbits/sec    3    350 KBytes\n[  4]   1.00-2.00   sec  89.9 MBytes   754 Mbits/sec  118    473 KBytes\n[  4]   2.00-3.00   sec   101 MBytes   848 Mbits/sec  184    586 KBytes\n[  4]   3.00-4.00   sec   104 MBytes   875 Mbits/sec  217    671 KBytes\n[  4]   4.00-5.00   sec   111 MBytes   935 Mbits/sec  175    772 KBytes\n[  4]   5.00-6.00   sec   100 MBytes   840 Mbits/sec  658    598 KBytes\n[  4]   6.00-7.00   sec   106 MBytes   890 Mbits/sec  742    668 KBytes\n[  4]   7.00-8.00   sec   102 MBytes   857 Mbits/sec  764    724 KBytes\n[  4]   8.00-9.00   sec  97.4 MBytes   817 Mbits/sec  1175    764 KBytes\n[  4]   9.00-10.00  sec   111 MBytes   934 Mbits/sec  1083    838 KBytes\n- - - - - - - - - - - - - - - - - - - - - - - - -\n[ ID] Interval           Transfer     Bandwidth       Retr\n[  4]   0.00-10.00  sec  1010 MBytes   848 Mbits/sec  5119             sender\n[  4]   0.00-10.00  sec  1008 MBytes   846 Mbits/sec                  receiver\n\niperf Done.\n</code></pre> <p>Modify the ingress bandwidth QoS for the first Pod:</p> <pre><code>kubectl annotate --overwrite  pod perf-4n4gt -n ls1 ovn.kubernetes.io/ingress_rate=30\n</code></pre> <p>Test the Pod bandwidth again from the second Pod:</p> <pre><code># iperf3 -c 10.66.0.12\nConnecting to host 10.66.0.12, port 5201\n[  4] local 10.66.0.14 port 52372 connected to 10.66.0.12 port 5201\n[ ID] Interval           Transfer     Bandwidth       Retr  Cwnd\n[  4]   0.00-1.00   sec  3.66 MBytes  30.7 Mbits/sec    2   76.1 KBytes\n[  4]   1.00-2.00   sec  3.43 MBytes  28.8 Mbits/sec    0    104 KBytes\n[  4]   2.00-3.00   sec  3.50 MBytes  29.4 Mbits/sec    0    126 KBytes\n[  4]   3.00-4.00   sec  3.50 MBytes  29.3 Mbits/sec    0    144 KBytes\n[  4]   4.00-5.00   sec  3.43 MBytes  28.8 Mbits/sec    0    160 KBytes\n[  4]   5.00-6.00   sec  3.43 MBytes  28.8 Mbits/sec    0    175 KBytes\n[  4]   6.00-7.00   sec  3.50 MBytes  29.3 Mbits/sec    0    212 KBytes\n[  4]   7.00-8.00   sec  3.68 MBytes  30.9 Mbits/sec    0    294 KBytes\n[  4]   8.00-9.00   sec  3.74 MBytes  31.4 Mbits/sec    0    398 KBytes\n[  4]   9.00-10.00  sec  3.80 MBytes  31.9 Mbits/sec    0    526 KBytes\n- - - - - - - - - - - - - - - - - - - - - - - - -\n[ ID] Interval           Transfer     Bandwidth       Retr\n[  4]   0.00-10.00  sec  35.7 MBytes  29.9 Mbits/sec    2             sender\n[  4]   0.00-10.00  sec  34.5 MBytes  29.0 Mbits/sec                  receiver\n\niperf Done.\n</code></pre>"},{"location":"en/guide/qos/#linux-netem-qos","title":"linux-netem QoS","text":"<p>Pod can use annotation below to config <code>linux-netem</code> type QoS\uff1a <code>ovn.kubernetes.io/latency</code>\u3001<code>ovn.kubernetes.io/limit</code> and <code>ovn.kubernetes.io/loss</code>\u3002</p> <ul> <li><code>ovn.kubernetes.io/latency</code>: Set the Pod traffic delay to an integer value in ms.</li> <li><code>ovn.kubernetes.io/limit</code>\uff1a Set the maximum number of packets that the <code>qdisc</code> queue can hold, and takes an integer value, such as 1000.</li> <li><code>ovn.kubernetes.io/loss</code>\uff1a Set packet loss probability, the value is float type, for example, the value is 20, then it is set 20% packet loss probability.</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/setup-options/","title":"Installation and Configuration Options","text":"<p>In One-Click Installation we use the default configuration for installation. Kube-OVN also supports more custom configurations, which can be configured in the installation script, or later by changing the parameters of individual components. This document will describe what these customization options do, and how to configure them.</p>"},{"location":"en/guide/setup-options/#built-in-network-settings","title":"Built-in Network Settings","text":"<p>Kube-OVN will configure two built-in Subnets during installation:</p> <ol> <li><code>default</code> Subnet, as the default subnet used by the Pod to assign IPs, with a default CIDR of <code>10.16.0.0/16</code> and a gateway of <code>10.16.0.1</code>.</li> <li>The <code>join</code> subnet, as a special subnet for network communication between the Node and Pod, has a default CIDR of <code>100.64.0.0/16</code> and a gateway of <code>100.64.0.1</code>.</li> </ol> <p>The configuration of these two subnets can be changed during installation via the installation scripts variables:</p> <pre><code>POD_CIDR=\"10.16.0.0/16\"\nPOD_GATEWAY=\"10.16.0.1\"\nJOIN_CIDR=\"100.64.0.0/16\"\nEXCLUDE_IPS=\"\"\n</code></pre> <p><code>EXCLUDE_IP</code> sets the address range for which <code>kube-ovn-controller</code> will not automatically assign from it, the format is: <code>192.168.10.20..192.168.10.30</code>.</p> <p>Note that in the Overlay case these two Subnets CIDRs cannot conflict with existing host networks and Service CIDRs.</p> <p>You can change the address range of both Subnets after installation by referring to Change Subnet CIDR and Change Join Subnet CIDR.</p>"},{"location":"en/guide/setup-options/#config-service-cidr","title":"Config Service CIDR","text":"<p>Since some of the iptables and routing rules set by <code>kube-proxy</code> will conflict with the rules set by Kube-OVN, Kube-OVN needs to know the CIDR of the service to set the corresponding rules correctly.</p> <p>This can be done by modifying the installation script:</p> <pre><code>SVC_CIDR=\"10.96.0.0/12\"  </code></pre> <p>You can also modify the args of the <code>kube-ovn-controller</code> Deployment after installation:</p> <pre><code>args:\n- --service-cluster-ip-range=10.96.0.0/12\n</code></pre>"},{"location":"en/guide/setup-options/#overlay-nic-selection","title":"Overlay NIC Selection","text":"<p>In the case of multiple NICs on a node, Kube-OVN will select the NIC corresponding to the Kubernetes Node IP as the NIC for cross-node communication between containers and establish the corresponding tunnel.</p> <p>If you need to select another NIC to create a container tunnel, you can change it in the installation script:</p> <pre><code>IFACE=eth1\n</code></pre> <p>This option supports regular expressions separated by commas, e.g. 'ens[a-z0-9],eth[a-z0-9]'.</p> <p>It can also be adjusted after installation by modifying the args of the <code>kube-ovn-cni</code> DaemonSet:</p> <pre><code>args:\n- --iface=eth1\n</code></pre> <p>If each machine has a different NIC name and there is no fixed pattern, you can use the node annotation <code>ovn.kubernetes.io/tunnel_interface</code> to configure each node one by one. This annotation will override the configuration of <code>iface</code>.</p> <pre><code>kubectl annotate node no1 ovn.kubernetes.io/tunnel_interface=ethx\n</code></pre>"},{"location":"en/guide/setup-options/#config-mtu","title":"Config MTU","text":"<p>Since Overlay encapsulation requires additional space, Kube-OVN will adjust the MTU of the container NIC based on the MTU of the selected NIC when creating the container NIC. By default, the Pod NIC MTU is the host NIC MTU - 100 on the Overlay Subnet, and the Pod NIC and host NIC have the same MTU on the Underlay Subnet.</p> <p>If you need to adjust the size of the MTU under the Overlay subnet, you can modify the parameters of the <code>kube-ovn-cni</code> DaemonSet:</p> <pre><code>args:\n- --mtu=1333\n</code></pre>"},{"location":"en/guide/setup-options/#global-traffic-mirroring-setting","title":"Global Traffic Mirroring Setting","text":"<p>When global traffic mirroring is enabled, Kube-OVN will create a <code>mirror0</code> virtual NIC on each node and copy all container network traffic from the current machine to that NIC\uff0c Users can perform traffic analysis with tcpdump and other tools. This function can be enabled in the installation script:</p> <pre><code>ENABLE_MIRROR=true\n</code></pre> <p>It can also be adjusted after installation by modifying the args of the <code>kube-ovn-cni</code> DaemonSet:</p> <pre><code>args:\n- --enable-mirror=true\n</code></pre> <p>The ability to mirror traffic is disabled in the default installation, if you need fine-grained traffic mirroring or need to mirror traffic to additional NICs please refer to Traffic Mirror.</p>"},{"location":"en/guide/setup-options/#lb-settings","title":"LB Settings","text":"<p>Kube-OVN uses L2 LB in OVN to implement service forwarding. In Overlay scenarios, users can choose to use <code>kube-proxy</code> for service traffic forwarding, in which case the LB function of Kube-OVN can be disabled to achieve better performance on the control plane and data plane.</p> <p>This feature can be configured in the installation script:</p> <pre><code>ENABLE_LB=false\n</code></pre> <p>It can also be configured after installation by changing the args of the <code>kube-ovn-controller</code> Deployment:</p> <pre><code>args:\n- --enable-lb=false\n</code></pre> <p>The LB feature is enabled in the default installation.</p> <p>The spec field <code>enableLb</code> has been added to the subnet crd definition since Kube-OVN v1.12.0 to migrate the LB function of Kube-OVN to the subnet level. You can set whether to enable the LB function based on different subnets. The <code>enable-lb</code> parameter in the <code>kube-ovn-controller</code> deployment is used as a global switch to control whether to create a load-balancer record. The <code>enableLb</code> parameter added in the subnet is used to control whether the subnet is associated with a load-balancer record. After the previous version is upgraded to v1.12.0, the <code>enableLb</code> parameter of the subnet will automatically inherit the value of the original global switch parameter.</p>"},{"location":"en/guide/setup-options/#networkpolicy-settings","title":"NetworkPolicy Settings","text":"<p>Kube-OVN uses ACLs in OVN to implement NetworkPolicy. Users can choose to disable the NetworkPolicy feature or use the Cilium Chain approach to implement NetworkPolicy using eBPF. In this case, the NetworkPolicy feature of Kube-OVN can be disabled to achieve better performance on the control plane and data plane.</p> <p>This feature can be configured in the installation script:</p> <pre><code>ENABLE_NP=false\n</code></pre> <p>It can also be configured after installation by changing the args of the <code>kube-ovn-controller</code> Deployment:</p> <pre><code>args:\n- --enable-np=false\n</code></pre> <p>NetworkPolicy is enabled by default.</p>"},{"location":"en/guide/setup-options/#eip-and-snat-settings","title":"EIP and SNAT Settings","text":"<p>If the EIP and SNAT capabilities are not required on the default VPC, users can choose to disable them to reduce the performance overhead of <code>kube-ovn-controller</code> in large scale cluster environments and improve processing speed.</p> <p>This feature can be configured in the installation script:</p> <pre><code>ENABLE_EIP_SNAT=false\n</code></pre> <p>It can also be configured after installation by changing the args of the <code>kube-ovn-controller</code> Deployment:</p> <pre><code>args:\n- --enable-eip-snat=false\n</code></pre> <p>EIP and SNAT is enabled by default. More information can refer to EIP and SNAT\u3002</p>"},{"location":"en/guide/setup-options/#centralized-gateway-ecmp-settings","title":"Centralized Gateway ECMP Settings","text":"<p>The centralized gateway supports two mode of high availability, primary-backup and ECMP. If you want to enable ECMP mode, you need to change the args of <code>kube-ovn-controller</code> Deployment:</p> <pre><code>args:\n- --enable-ecmp=true </code></pre> <p>Centralized gateway default installation under the primary-backup mode, more gateway-related content please refer to Config Subnet.</p> <p>The spec field <code>enableEcmp</code> has been added to the subnet crd definition since Kube-OVN v1.12.0 to migrate the ECMP switch to the subnet level. You can set whether to enable ECMP mode based on different subnets. The <code>enable-ecmp</code> parameter in the <code>kube-ovn-controller</code> deployment is no longer used. After the previous version is upgraded to v1.12.0, the subnet switch will automatically inherit the value of the original global switch parameter.</p>"},{"location":"en/guide/setup-options/#kubevirt-vm-fixed-address-settings","title":"Kubevirt VM Fixed Address Settings","text":"<p>For VM instances created by Kubevirt, <code>kube-ovn-controller</code> can assign and manage IP addresses in a similar way to the StatefulSet Pod. This allows VM instances address fixed during start-up, shutdown, upgrade, migration, and other operations throughout their lifecycle, making them more compatible with the actual virtualization user experience.</p> <p>This feature is enabled by default after v1.10.6. To disable this feature, you need to change the following args in the <code>kube-ovn-controller</code> Deployment:</p> <pre><code>args:\n- --keep-vm-ip=false\n</code></pre>"},{"location":"en/guide/setup-options/#cni-settings","title":"CNI Settings","text":"<p>By default, Kube-OVN installs the CNI binary in the <code>/opt/cni/bin</code> directory and the CNI configuration file <code>01-kube-ovn.conflist</code> in the <code>/etc/cni/net.d</code> directory. If you need to change the installation location and the priority of the CNI configuration file, you can modify the following parameters of the installation script.</p> <pre><code>CNI_CONF_DIR=\"/etc/cni/net.d\"\nCNI_BIN_DIR=\"/opt/cni/bin\"\nCNI_CONFIG_PRIORITY=\"01\"\n</code></pre> <p>Or change the Volume mount and args of the <code>kube-ovn-cni</code> DaemonSet after installation:</p> <pre><code>volumes:\n- name: cni-conf\nhostPath:\npath: \"/etc/cni/net.d\"\n- name: cni-bin\nhostPath:\npath:\"/opt/cni/bin\"\n...\nargs:\n- --cni-conf-name=01-kube-ovn.conflist\n</code></pre>"},{"location":"en/guide/setup-options/#tunnel-type-settings","title":"Tunnel Type Settings","text":"<p>The default encapsulation mode of Kube-OVN Overlay is Geneve, if you want to change it to Vxlan or STT, please adjust the following parameters in the installation script:</p> <pre><code>TUNNEL_TYPE=\"vxlan\"\n</code></pre> <p>Or change the environment variables of <code>ovs-ovn</code> DaemonSet after installation:</p> <pre><code>env:\n- name: TUNNEL_TYPE\nvalue: \"vxlan\"\n</code></pre> <p>If you need to use the STT tunnel and need to compile additional kernel modules for ovs, please refer to Performance Tunning\u3002</p> <p>Please refer to Tunneling Protocol Selection for the differences between the different protocols in practice.</p>"},{"location":"en/guide/setup-options/#ssl-settings","title":"SSL Settings","text":"<p>The OVN DB API interface supports SSL encryption to secure the connection. To enable it, adjust the following parameters in the installation script:</p> <pre><code>ENABLE_SSL=true\n</code></pre> <p>The SSL is disabled by default.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/static-ip-mac/","title":"Fixed Addresses","text":"<p>By default, Kube-OVN randomly assigns IPs and Macs based on the Subnet to which the Pod's Namespace belongs. For workloads that require fixed addresses, Kube-OVN provides multiple methods of fixing addresses depending on the scenario.</p> <ul> <li>Single Pod fixed IP/Mac.</li> <li>Workload IP Pool to specify fixed addresses.</li> <li>StatefulSet fixed address.</li> <li>KubeVirt VM fixed address.</li> </ul>"},{"location":"en/guide/static-ip-mac/#single-pod-fixed-ipmac","title":"Single Pod Fixed IP/Mac","text":"<p>You can specify the IP/Mac required for the Pod by annotation when creating the Pod. The <code>kube-ovn-controller</code> will skip the address random assignment phase and use the specified address directly after conflict detection, as follows:</p> <pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nname: ippool\nlabels:\napp: ippool\nspec:\nreplicas: 2\nselector:\nmatchLabels:\napp: ippool\ntemplate:\nmetadata:\nlabels:\napp: ippool\nannotations:\novn.kubernetes.io/ip_pool: 10.16.0.15,10.16.0.16,10.16.0.17 // for dualstack ippool use semicolon to separate addresses 10.16.0.15,fd00:10:16::000E;10.16.0.16,fd00:10:16::0\nspec:\ncontainers:\n- name: ippool\nimage: docker.io/library/nginx:alpine\n</code></pre> <p>The following points need to be noted when using annotation.</p> <ol> <li>The IP/Mac used cannot conflict with an existing IP/Mac.</li> <li>The IP must be in the CIDR range of the Subnet it belongs to.</li> <li>You can specify only IP or Mac. When you specify only one, the other one will be assigned randomly.</li> </ol>"},{"location":"en/guide/static-ip-mac/#workload-ip-pool","title":"Workload IP Pool","text":"<p>Kube-OVN supports setting fixed IPs for Workloads (Deployment/StatefulSet/DaemonSet/Job/CronJob) via annotation <code>ovn.kubernetes.io/ip_pool</code>. <code>kube-ovn-controllerr</code> will automatically select the IP specified in <code>ovn.kubernetes.io/ip_pool</code> and perform conflict detection.</p> <p>The Annotation of the IP Pool needs to be added to the <code>annotation</code> field in the <code>template</code>. In addition to Kubernetes built-in workload types, other user-defined workloads can also be assigned fixed addresses using the same approach.</p>"},{"location":"en/guide/static-ip-mac/#deployment-with-fixed-ips","title":"Deployment With Fixed IPs","text":"<pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nnamespace: ls1\nname: starter-backend\nlabels:\napp: starter-backend\nspec:\nreplicas: 2\nselector:\nmatchLabels:\napp: starter-backend\ntemplate:\nmetadata:\nlabels:\napp: starter-backend\nannotations:\novn.kubernetes.io/ip_pool: 10.16.0.15,10.16.0.16,10.16.0.17 // for dualstack ippool use semicolon to separate addresses 10.16.0.15,fd00:10:16::000E;10.16.0.16,fd00:10:16::000F;10.16.0.17,fd00:10:16::0010\nspec:\ncontainers:\n- name: backend\nimage: docker.io/library/nginx:alpine\n</code></pre> <p>Using a fixed IP for Workload requires the following:</p> <ol> <li>The IP in <code>ovn.kubernetes.io/ip_pool</code> should belong to the CIDR of the Subnet.</li> <li>The IP in <code>ovn.kubernetes.io/ip_pool</code> cannot conflict with an IP already in use.</li> <li>When the number of IPs in <code>ovn.kubernetes.io/ip_pool</code> is less than the number of replicas, the extra Pods will not be created. You need to adjust the number of IPs in <code>ovn.kubernetes.io/ip_pool</code> according to the update policy of the workload and the scaling plan.</li> </ol>"},{"location":"en/guide/static-ip-mac/#statefulset-fixed-address","title":"StatefulSet Fixed Address","text":"<p>StatefulSet supports fixed IP by default, and like other Workload, you can use <code>ovn.kubernetes.io/ip_pool</code> to specify the range of IP used by a Pod.</p> <p>Since StatefulSet is mostly used for stateful services, which have higher requirements for fixed addresses, Kube-OVN has made special enhancements:</p> <ol> <li>Pods are assigned IPs in <code>ovn.kubernetes.io/ip_pool</code> in order. For example, if the name of the StatefulSet is web, web-0 will use the first IP in <code>ovn.kubernetes.io/ip_pool</code>, web-1 will use the second IP, and so on.</li> <li>The logical_switch_port in the OVN is not deleted during update or deletion of the StatefulSet Pod, and the newly generated Pod directly reuses the old logical port information. Pods can therefore reuse IP/Mac and other network information to achieve similar state retention as StatefulSet Volumes.</li> <li>Based on the capabilities of 2, for StatefulSet without the <code>ovn.kubernetes.io/ip_pool</code> annotation, a Pod is randomly assigned an IP/Mac when it is first generated, and then the network information remains fixed for the lifetime of the StatefulSet.</li> </ol>"},{"location":"en/guide/static-ip-mac/#statefulset-example","title":"StatefulSet Example","text":"<pre><code>apiVersion: apps/v1\nkind: StatefulSet\nmetadata:\nname: web\nspec:\nserviceName: \"nginx\"\nreplicas: 2\nselector:\nmatchLabels:\napp: nginx\ntemplate:\nmetadata:\nlabels:\napp: nginx\nspec:\ncontainers:\n- name: nginx\nimage: docker.io/library/nginx:alpine\nports:\n- containerPort: 80\nname: web\n</code></pre> <p>You can try to delete the Pod under StatefulSet to observe if the Pod IP changes.</p>"},{"location":"en/guide/static-ip-mac/#kubevirt-vm-fixed-address","title":"KubeVirt VM Fixed Address","text":"<p>For VM instances created by KubeVirt, <code>kube-ovn-controller</code> can assign and manage IP addresses in a similar way to the StatefulSet Pod. This allows VM instances address fixed during start-up, shutdown, upgrade, migration, and other operations throughout their lifecycle, making them more compatible with the actual virtualization user experience.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/subnet/","title":"Config Subnet","text":"<p>Subnet is a core concept and basic unit of use in Kube-OVN, and Kube-OVN organizes IP and network configuration in terms of Subnet. Each Namespace can belong to a specific Subnet, and Pods under the Namespace automatically obtain IPs from the Subnet they belong to and share the network configuration (CIDR, gateway type, access control, NAT control, etc.).</p> <p>Unlike other CNI implementations where each node is bound to a subnet, in Kube-OVN the Subnet is a global level virtual network configuration, and the addresses of one Subnet can be distributed on any node.</p> <p></p> <p>There are some differences in the usage and configuration of Overlay and Underlay Subnets, and this document will describe the common configurations and differentiated features of the different types of Subnets.</p>"},{"location":"en/guide/subnet/#default-subnet","title":"Default Subnet","text":"<p>To make it easier for users to get started quickly, Kube-OVN has a built-in default Subnet, all Namespaces that do not explicitly declare subnet affiliation are automatically assigned IPs from the default subnet and the network information. The configuration of this Subnet is specified at installation time, you can refer to Built-in Network Settings for more details. To change the CIDR of the default Subnet after installation please refer to Change Subnet CIDR.</p> <p>In Overlay mode, the default Subnet uses a distributed gateway and NAT translation for outbound traffic, which behaves much the same as the Flannel's default behavior, allowing users to use most of the network features without additional configuration.</p> <p>In Underlay mode, the default Subnet uses the physical gateway as the outgoing gateway and enables arping to check network connectivity.</p>"},{"location":"en/guide/subnet/#check-the-default-subnet","title":"Check the Default Subnet","text":"<p>The <code>default</code> field in the default Subnet spec is set to <code>true</code>, and there is only one default Subnet in a cluster, named <code>ovn-default</code>.</p> <pre><code># kubectl get subnet ovn-default -o yaml\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n  creationTimestamp: \"2019-08-06T09:33:43Z\"\ngeneration: 1\nname: ovn-default\n  resourceVersion: \"1571334\"\nselfLink: /apis/kubeovn.io/v1/subnets/ovn-default\n  uid: 7e2451f8-fb44-4f7f-b3e0-cfd27f6fd5d6\nspec:\n  cidrBlock: 10.16.0.0/16\n  default: true\nexcludeIps:\n  - 10.16.0.1\n  gateway: 10.16.0.1\n  gatewayType: distributed\n  natOutgoing: true\nprivate: false\nprotocol: IPv4\n</code></pre>"},{"location":"en/guide/subnet/#join-subnet","title":"Join Subnet","text":"<p>In the Kubernetes network specification, it is required that Nodes can communicate directly with all Pods. To achieve this in Overlay network mode, Kube-OVN creates a <code>join</code> Subnet and creates a virtual NIC <code>ovn0</code> at each node that connect to the <code>join</code> subnet, through which the nodes and Pods can communicate with each other.</p> <p>The configuration of this Subnet is specified at installation time, you can refer to Built-in Network Settings for more details. To change the CIDR of the Join Subnet after installation please refer to Change Join CIDR.</p>"},{"location":"en/guide/subnet/#check-the-join-subnet","title":"Check the Join Subnet","text":"<p>The default name of this subnet is <code>join</code>. There is generally no need to make changes to the network configuration except the CIDR.</p> <pre><code># kubectl get subnet join -o yaml\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n  creationTimestamp: \"2019-08-06T09:33:43Z\"\ngeneration: 1\nname: join\n  resourceVersion: \"1571333\"\nselfLink: /apis/kubeovn.io/v1/subnets/join\n  uid: 9c744810-c678-4d50-8a7d-b8ec12ef91b8\nspec:\n  cidrBlock: 100.64.0.0/16\n  default: false\nexcludeIps:\n  - 100.64.0.1\n  gateway: 100.64.0.1\n  gatewayNode: \"\"\ngatewayType: \"\"\nnatOutgoing: false\nprivate: false\nprotocol: IPv4\n</code></pre> <p>Check the ovn0 NIC at the node:</p> <pre><code># ifconfig ovn0\novn0: flags=4163&lt;UP,BROADCAST,RUNNING,MULTICAST&gt;  mtu 1420\ninet 100.64.0.4  netmask 255.255.0.0  broadcast 100.64.255.255\n        inet6 fe80::800:ff:fe40:5  prefixlen 64  scopeid 0x20&lt;link&gt;\n        ether 0a:00:00:40:00:05  txqueuelen 1000  (Ethernet)\nRX packets 18  bytes 1428 (1.3 KiB)\nRX errors 0  dropped 0  overruns 0  frame 0\nTX packets 19  bytes 1810 (1.7 KiB)\nTX errors 0  dropped 0 overruns 0  carrier 0  collisions 0\n</code></pre>"},{"location":"en/guide/subnet/#create-custom-subnets","title":"Create Custom Subnets","text":"<p>Here we describe the basic operation of how to create a Subnet and associate it with a Namespace, for more advanced configuration, please refer to the subsequent content.</p>"},{"location":"en/guide/subnet/#create-subnet","title":"Create Subnet","text":"<pre><code>cat &lt;&lt;EOF | kubectl create -f -\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n  name: subnet1\nspec:\n  protocol: IPv4\n  cidrBlock: 10.66.0.0/16\n  excludeIps:\n  - 10.66.0.1..10.66.0.10\n  - 10.66.0.101..10.66.0.151\n  gateway: 10.66.0.1\n  gatewayType: distributed\n  natOutgoing: true\n  routeTable: \"\"\n  namespaces:\n  - ns1\n  - ns2\nEOF\n</code></pre> <ul> <li><code>cidrBlock</code>: Subnet CIDR range, different Subnet CIDRs under the same VPC cannot overlap.</li> <li><code>excludeIps</code>: The address list is reserved so that the container network will not automatically assign addresses in the list, which can be used as a fixed IP address assignment segment or to avoid conflicts with existing devices in the physical network in Underlay mode.</li> <li><code>gateway</code>\uff1aFor this subnet gateway address, Kube-OVN will automatically assign the corresponding logical gateway in Overlay mode, and the address should be the underlying physical gateway address in Underlay mode.</li> <li><code>namespaces</code>: Bind the list of Namespace for this Subnet. Pods under the Namespace will be assigned addresses from the current Subnet after binding.</li> <li><code>routeTable</code>: Associate the route table, default is main table, route table definition please defer to Static Routes</li> </ul>"},{"location":"en/guide/subnet/#create-pod-in-the-subnet","title":"Create Pod in the Subnet","text":"<pre><code># kubectl create ns ns1\nnamespace/ns1 created\n\n# kubectl run nginx --image=docker.io/library/nginx:alpine -n ns1\ndeployment.apps/nginx created\n\n# kubectl get pod -n ns1 -o wide\nNAME                     READY   STATUS    RESTARTS   AGE   IP           NODE    NOMINATED NODE   READINESS GATES\nnginx-74d5899f46-n8wtg   1/1     Running   0          10s   10.66.0.11   node1   &lt;none&gt;           &lt;none&gt;\n</code></pre>"},{"location":"en/guide/subnet/#overlay-subnet-gateway-settings","title":"Overlay Subnet Gateway Settings","text":"<p>This feature only works for Overlay mode Subnets, Underlay type Subnets need to use the underlying physical gateway to access the external network.</p> <p>Pods under the Overlay Subnet need to access the external network through a gateway, and Kube-OVN currently supports two types of gateways: distributed gateway and centralized gateway which can be changed in the Subnet spec.</p> <p>Both types of gateways support the <code>natOutgoing</code> setting, which allows the user to choose whether snat is required when the Pod accesses the external network.</p>"},{"location":"en/guide/subnet/#distributed-gateway","title":"Distributed Gateway","text":"<p>The default type of gateway for the Subnet, each node will act as a gateway for the pod on the current node to access the external network. The packets from container will flow into the host network stack from the local <code>ovn0</code> NIC, and then forwarding the network according to the host's routing rules. When <code>natOutgoing</code> is <code>true</code>, the Pod will use the IP of the current host when accessing the external network.</p> <p></p> <p>Example of a Subnet, where the <code>gatewayType</code> field is <code>distributed</code>:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: distributed\nspec:\nprotocol: IPv4\ncidrBlock: 10.166.0.0/16\ndefault: false\nexcludeIps:\n- 10.166.0.1\ngateway: 10.166.0.1\ngatewayType: distributed\nnatOutgoing: true\n</code></pre>"},{"location":"en/guide/subnet/#centralized-gateway","title":"Centralized Gateway","text":"<p>If you want traffic within the Subnet to access the external network using a fixed IP for security operations such as auditing and whitelisting, you can set the gateway type in the Subnet to centralized. In centralized gateway mode, packets from Pods accessing the external network are first routed to the <code>ovn0</code> NIC of a specific nodes, and then outbound through the host's routing rules. When <code>natOutgoing</code> is <code>true</code>, the Pod will use the IP of a specific nodes when accessing the external network.</p> <p>The centralized gateway example is as follows, where the <code>gatewayType</code> field is <code>centralized</code> and <code>gatewayNode</code> is the NodeName of the particular machine in Kubernetes.</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: centralized\nspec:\nprotocol: IPv4\ncidrBlock: 10.166.0.0/16\ndefault: false\nexcludeIps:\n- 10.166.0.1\ngateway: 10.166.0.1\ngatewayType: centralized\ngatewayNode: \"node1,node2\"\nnatOutgoing: true\n</code></pre> <ul> <li>If a centralized gateway wants to specify a specific NIC of a machine for outbound networking, <code>gatewayNode</code> format can be changed to <code>kube-ovn-worker:172.18.0.2, kube-ovn-control-plane:172.18.0.3</code>.</li> <li>The centralized gateway defaults to primary-backup mode, with only the primary node performing traffic forwarding.   If you need to switch to ECMP mode, please refer to ECMP Settings.</li> <li>The spec field <code>enableEcmp</code> has been added to the subnet crd definition since Kube-OVN v1.12.0 to migrate the ECMP switch to the subnet level. You can set whether to enable ECMP mode based on different subnets. The <code>enable-ecmp</code> parameter in the <code>kube-ovn-controller</code> deployment is no longer used. After the previous version is upgraded to v1.12.0, the subnet switch will automatically inherit the value of the original global switch parameter.</li> </ul>"},{"location":"en/guide/subnet/#subnet-acl","title":"Subnet ACL","text":"<p>For scenarios with fine-grained ACL control, Subnet of Kube-OVN provides ACL to enable fine-grained rules.</p> <p>The ACL rules in Subnet are the same as the ACL rules in OVN, and you can refer to ovn-nb ACL Table for more details. The supported filed in <code>match</code> can refer to ovn-sb Logical Flow Table.</p> <p>Example of an ACL rule that allows Pods with IP address <code>10.10.0.2</code> to access all addresses, but does not allow other addresses to access itself, is as follows:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: acl\nspec:\nacls:\n- action: drop\ndirection: to-lport\nmatch: ip4.dst == 10.10.0.2 &amp;&amp; ip\npriority: 1002\n- action: allow-related\ndirection: from-lport\nmatch: ip4.src == 10.10.0.2 &amp;&amp; ip\npriority: 1002\ncidrBlock: 10.10.0.0/24\n</code></pre>"},{"location":"en/guide/subnet/#subnet-isolation","title":"Subnet Isolation","text":"<p>The function of Subnet ACL can cover the function of Subnet isolation with better flexibility, we recommend using Subnet ACL to do the corresponding configuration.</p> <p>By default the Subnets created by Kube-OVN can communicate with each other, and Pods can also access external networks through the gateway.</p> <p>To control access between Subnets, set <code>private</code> to true in the subnet spec, and the Subnet will be isolated from other Subnets and external networks and can only communicate within the Subnet. If you want to open a whitelist, you can set it by <code>allowSubnets</code>. The CIDRs in <code>allowSubnets</code> can access the Subnet bidirectionally.</p>"},{"location":"en/guide/subnet/#enable-subnet-isolation-examples","title":"Enable Subnet Isolation Examples","text":"<pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: private\nspec:\nprotocol: IPv4\ndefault: false\nnamespaces:\n- ns1\n- ns2\ncidrBlock: 10.69.0.0/16\nprivate: true\nallowSubnets:\n- 10.16.0.0/16\n- 10.18.0.0/16\n</code></pre>"},{"location":"en/guide/subnet/#underlay-settings","title":"Underlay Settings","text":"<p>This part of the feature is only available for Underlay type Subnets.</p> <ul> <li><code>vlan</code>: If an Underlay network is used, this field is used to control which Vlan CR the Subnet is bound to. This option defaults to the empty string, meaning that the Underlay network is not used.</li> <li><code>logicalGateway</code>: Some Underlay environments are pure Layer 2 networks, with no physical Layer 3 gateway. In this case a virtual gateway can be set up with the OVN to connect the Underlay and Overlay networks. The default value is: <code>false</code>.</li> </ul>"},{"location":"en/guide/subnet/#gateway-check-settings","title":"Gateway Check Settings","text":"<p>By default <code>kube-ovn-cni</code> will request the gateway using ICMP or ARP protocol after starting the Pod and wait for the return to verify that the network is working properly. Some Underlay environment gateways cannot respond to ARP requests, or scenarios that do not require external connectivity, the checking can be disabled .</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: disable-gw-check\nspec:\ndisableGatewayCheck: true\n</code></pre>"},{"location":"en/guide/subnet/#multicast-snoop-setting","title":"Multicast-Snoop Setting","text":"<p>By default, if a Pod in a subnet sends a multicast packet, OVN's default behavior is to broadcast the multicast packet to all Pods in the subnet. If turned on the subnet's multicast snoop switch, OVN will forward based on the multicast table <code>Multicast_Group</code> in the <code>South Database</code> instead of broadcasting.</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: sample1\nspec:\nenableMulticastSnoop: true\n</code></pre>"},{"location":"en/guide/subnet/#other-advanced-settings","title":"Other Advanced Settings","text":"<ul> <li>Configure IPPool</li> <li>Default VPC NAT Policy Rule</li> <li>Manage QoS</li> <li>Manage Multiple Interface</li> <li>DHCP</li> <li>External Gateway</li> <li>Cluster Inter-Connection with OVN-IC</li> <li>VIP Reservation</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/vpc-qos/","title":"VPC QoS","text":"<p>Kube-OVN supports using QoSPolicy CRD to limit the traffic rate of custom VPC.</p>"},{"location":"en/guide/vpc-qos/#eip-qos","title":"EIP QoS","text":"<p>Limit the speed of EIP to 1Mbps and the priority to 1, and <code>shared=false</code> here means that this QoSPolicy can only be used for this EIP and support dynamically modifying QoSPolicy to change QoS rules.</p> <p>The QoSPolicy configuration is as follows:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: QoSPolicy\nmetadata:\nname: qos-eip-example\nspec:\nshared: false\nbindingType: EIP\nbandwidthLimitRules:\n- name: eip-ingress\nrateMax: \"1\" # Mbps\nburstMax: \"1\" # Mbps\npriority: 1\ndirection: ingress\n- name: eip-egress\nrateMax: \"1\" # Mbps\nburstMax: \"1\" # Mbps\npriority: 1\ndirection: egress\n</code></pre> <p>The IptablesEIP configuration is as follows:</p> <pre><code>kind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-1\nspec:\nnatGwDp: gw1\nqosPolicy: qos-eip-example\n</code></pre> <p>The value of <code>.spec.qosPolicy</code> supports being specified during creation and also supports modification after creation.</p>"},{"location":"en/guide/vpc-qos/#view-eips-with-qos-enabled","title":"View EIPs with QoS enabled","text":"<p>View the corresponding EIPs that have been set up using <code>label</code>:</p> <pre><code># kubectl get eip  -l ovn.kubernetes.io/qos=qos-eip-example\nNAME    IP             MAC                 NAT   NATGWDP   READY\neip-1   172.18.11.24   00:00:00:34:41:0B   fip   gw1       true\n</code></pre>"},{"location":"en/guide/vpc-qos/#qos-for-vpc-natgw-net1-nic","title":"QoS for VPC NATGW net1 NIC","text":"<p>Limit the speed of the net1 NIC on VPC NATGW to 10Mbps and set the priority to 3. Here <code>shared=true</code>, which means that this QoSPolicy can be used by multiple resources at the same time, and does not allow the modification of the contents of the QoSPolicy in this scenario.</p> <p>The QoSPolicy configuration is as follows:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: QoSPolicy\nmetadata:\nname: qos-natgw-example\nspec:\nshared: true\nbindingType: NATGW\nbandwidthLimitRules:\n- name: net1-ingress\ninterface: net1\nrateMax: \"10\" # Mbps\nburstMax: \"10\" # Mbps\npriority: 3\ndirection: ingress\n- name: net1-egress\ninterface: net1\nrateMax: \"10\" # Mbps\nburstMax: \"10\" # Mbps\npriority: 3\ndirection: egress\n</code></pre> <p>The VpcNatGateway configuration is as follows:</p> <pre><code>kind: VpcNatGateway\napiVersion: kubeovn.io/v1\nmetadata:\nname: gw1\nspec:\nvpc: test-vpc-1\nsubnet: net1\nlanIp: 10.0.1.254\nqosPolicy: qos-natgw-example\nselector:\n- \"kubernetes.io/hostname: kube-ovn-worker\"\n- \"kubernetes.io/os: linux\"\n</code></pre> <p>The value of <code>.spec.qosPolicy</code> supports both creation and subsequent modification.</p>"},{"location":"en/guide/vpc-qos/#qos-for-specific-traffic-on-net1-nic","title":"QoS for specific traffic on net1 NIC","text":"<p>Limit the specific traffic on net1 NIC to 5Mbps and set the priority to 2. Here <code>shared=true</code>, which means that this QoSPolicy can be used by multiple resources at the same time, and does not allow the modification of the contents of the QoSPolicy in this scenario.</p> <p>The QoSPolicy configuration is as follows:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: QoSPolicy\nmetadata:\nname: qos-natgw-example\nspec:\nshared: true\nbindingType: NATGW\nbandwidthLimitRules:\n- name: net1-extip-ingress\ninterface: net1\nrateMax: \"5\" # Mbps\nburstMax: \"5\" # Mbps\npriority: 2\ndirection: ingress\nmatchType: ip\nmatchValue: src 172.18.11.22/32\n- name: net1-extip-egress\ninterface: net1\nrateMax: \"5\" # Mbps\nburstMax: \"5\" # Mbps\npriority: 2\ndirection: egress\nmatchType: ip\nmatchValue: dst 172.18.11.23/32\n</code></pre> <p>The VpcNatGateway configuration is as follows:</p> <pre><code>kind: VpcNatGateway\napiVersion: kubeovn.io/v1\nmetadata:\nname: gw1\nspec:\nvpc: test-vpc-1\nsubnet: net1\nlanIp: 10.0.1.254\nqosPolicy: qos-natgw-example\nselector:\n- \"kubernetes.io/hostname: kube-ovn-worker\"\n- \"kubernetes.io/os: linux\"\n</code></pre>"},{"location":"en/guide/vpc-qos/#view-natgws-with-qos-enabled","title":"View NATGWs with QoS enabled","text":"<p>View the corresponding NATGWs that have been set up using <code>label</code>:</p> <pre><code># kubectl get vpc-nat-gw  -l ovn.kubernetes.io/qos=qos-natgw-example\nNAME   VPC          SUBNET   LANIP\ngw1    test-vpc-1   net1     10.0.1.254\n</code></pre>"},{"location":"en/guide/vpc-qos/#view-qos-rules","title":"View QoS rules","text":"<pre><code># kubectl get qos -A\nNAME                SHARED   BINDINGTYPE\nqos-eip-example     false    EIP\nqos-natgw-example   true     NATGW\n</code></pre>"},{"location":"en/guide/vpc-qos/#limitations","title":"Limitations","text":"<ul> <li>QoSPolicy can only be deleted when it is not in use. Therefore, before deleting the QoSPolicy, please check the EIP and NATGW that have enabled QoS, and remove their <code>spec.qosPolicy</code> configuration.</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/vpc/","title":"Config VPC","text":"<p>Kube-OVN supports multi-tenant isolation level VPC networks. Different VPC networks are independent of each other and can be configured separately with Subnet CIDRs, routing policies, security policies, outbound gateways, EIP, etc.</p> <p>VPC is mainly used in scenarios where there requires strong isolation of multi-tenant networks and some Kubernetes networking features conflict under multi-tenant networks. For example, node and pod access, NodePort functionality, network access-based health checks, and DNS capabilities are not supported in multi-tenant network scenarios at this time. In order to facilitate common Kubernetes usage scenarios, Kube-OVN has a special design for the default VPC where the Subnet under the VPC can meet the Kubernetes specification. The custom VPC supports static routing, EIP and NAT gateways as described in this document. Common isolation requirements can be achieved through network policies and Subnet ACLs under the default VPC, so before using a custom VPC, please make sure whether you need VPC-level isolation and understand the limitations under the custom VPC. For Underlay subnets, physical switches are responsible for data-plane forwarding, so VPCs cannot isolate Underlay subnets.</p> <p></p>"},{"location":"en/guide/vpc/#creating-custom-vpcs","title":"Creating Custom VPCs","text":"<p>Create two VPCs:</p> <pre><code>kind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: test-vpc-1\nspec:\nnamespaces:\n- ns1\n---\nkind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: test-vpc-2\nspec:\nnamespaces:\n- ns2\n</code></pre> <ul> <li><code>namespaces</code>: Limit which namespaces can use this VPC. If empty, all namespaces can use this VPC.</li> </ul> <p>Create two Subnets, belonging to two different VPCs and having the same CIDR:</p> <pre><code>kind: Subnet\napiVersion: kubeovn.io/v1\nmetadata:\nname: net1\nspec:\nvpc: test-vpc-1\ncidrBlock: 10.0.1.0/24\nprotocol: IPv4\nnamespaces:\n- ns1\n---\nkind: Subnet\napiVersion: kubeovn.io/v1\nmetadata:\nname: net2\nspec:\nvpc: test-vpc-2\ncidrBlock: 10.0.1.0/24\nprotocol: IPv4\nnamespaces:\n- ns2\n</code></pre> <p>Create Pods under two separate Namespaces:</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nannotations:\novn.kubernetes.io/logical_switch: net1\nnamespace: ns1\nname: vpc1-pod\nspec:\ncontainers:\n- name: vpc1-pod\nimage: docker.io/library/nginx:alpine\n---\napiVersion: v1\nkind: Pod\nmetadata:\nannotations:\novn.kubernetes.io/logical_switch: net2\nnamespace: ns2\nname: vpc2-pod\nspec:\ncontainers:\n- name: vpc2-pod\nimage: docker.io/library/nginx:alpine\n</code></pre> <p>After running successfully, you can observe that the two Pod addresses belong to the same CIDR, but the two Pods cannot access each other because they are running on different tenant VPCs.</p>"},{"location":"en/guide/vpc/#custom-vpc-pod-supports-livenessprobe-and-readinessprobe","title":"Custom VPC Pod supports livenessProbe and readinessProbe","text":"<p>Since the Pods under the custom VPC do not communicate with the network of the node, the probe packets sent by the kubelet cannot reach the Pods in the custom VPC. Kube-OVN uses TProxy to redirect the detection packets sent by kubelet to Pods in the custom VPC to achieve this function.</p> <p>The configuration method is as follows, add the parameter <code>--enable-tproxy=true</code> in Daemonset <code>kube-ovn-cni</code>:</p> <pre><code>spec:\ntemplate:\nspec:\ncontainers:\n- args:\n- --enable-tproxy=true\n</code></pre> <p>Restrictions for this feature:</p> <ol> <li>When Pods under different VPCs have the same IP under the same node, the detection function fails.</li> <li>Currently, only <code>tcpSocket</code> and <code>httpGet</code> are supported.</li> </ol>"},{"location":"en/guide/vpc/#create-vpc-nat-gateway","title":"Create VPC NAT Gateway","text":"<p>Subnets under custom VPCs do not support distributed gateways and centralized gateways under default VPCs.</p> <p>Pod access to the external network within the VPC requires a VPC gateway, which bridges the physical and tenant networks and provides floating IP, SNAT and DNAT capabilities.</p> <p>The VPC gateway function relies on Multus-CNI function, please refer to multus-cni.</p>"},{"location":"en/guide/vpc/#configuring-the-external-network","title":"Configuring the External Network","text":"<pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: ovn-vpc-external-network\nspec:\nprotocol: IPv4\nprovider: ovn-vpc-external-network.kube-system\ncidrBlock: 192.168.0.0/24\ngateway: 192.168.0.1  # IP address of the physical gateway\nexcludeIps:\n- 192.168.0.1..192.168.0.10\n---\napiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: ovn-vpc-external-network\nnamespace: kube-system\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.0\",\n\"type\": \"macvlan\",\n\"master\": \"eth1\",\n\"mode\": \"bridge\",\n\"ipam\": {\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"ovn-vpc-external-network.kube-system\"\n}\n}'\n</code></pre> <ul> <li>This Subnet is used to manage the available external addresses and the address will be allocated to VPC NAT Gateway through Macvlan, so please communicate with your network management to give you the available physical segment IPs.</li> <li>The VPC gateway uses Macvlan for physical network configuration, and <code>master</code> of <code>NetworkAttachmentDefinition</code> should be the NIC name of the corresponding physical network NIC.</li> <li><code>name</code>: External network name.</li> </ul> <p>For macvlan mode, the nic will send packets directly through that node NIC, relying on the underlying network devices for L2/L3 level forwarding capabilities. You need to configure the corresponding gateway, Vlan and security policy in the underlying network device in advance.</p> <ol> <li>For OpenStack VM environments, you need to turn off <code>PortSecurity</code> on the corresponding network port.</li> <li>For VMware vSwitch networks, <code>MAC Address Changes</code>, <code>Forged Transmits</code> and <code>Promiscuous Mode Operation</code> should be set to <code>allow</code>.</li> <li>For Hyper-V virtualization,  <code>MAC Address Spoofing</code> should be enabled in VM nic advanced features.</li> <li>Public clouds, such as AWS, GCE, AliCloud, etc., do not support user-defined Mac, so they cannot support Macvlan mode network.</li> <li>Due to the limitations of Macvlan, the Macvlan sub-interface cannot access the parent interface address.</li> <li>If the physical network card corresponds to a switch interface in Trunk mode, a sub-interface needs to be created on the network card and provided to Macvlan for use.</li> </ol>"},{"location":"en/guide/vpc/#enabling-the-vpc-gateway","title":"Enabling the VPC Gateway","text":"<p>VPC gateway functionality needs to be enabled via <code>ovn-vpc-nat-gw-config</code> under <code>kube-system</code>:</p> <pre><code>---\nkind: ConfigMap\napiVersion: v1\nmetadata:\nname: ovn-vpc-nat-config\nnamespace: kube-system\ndata:\nimage: docker.io/kubeovn/vpc-nat-gateway:v1.13.0\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\nname: ovn-vpc-nat-gw-config\nnamespace: kube-system\ndata:\nenable-vpc-nat-gw: 'true'\n</code></pre> <ul> <li><code>image</code>: The image used by the Gateway Pod.</li> <li><code>enable-vpc-nat-gw</code>: Controls whether the VPC Gateway feature is enabled.</li> </ul>"},{"location":"en/guide/vpc/#create-vpc-gateway-and-set-the-default-route","title":"Create VPC Gateway and Set the Default Route","text":"<pre><code>kind: VpcNatGateway\napiVersion: kubeovn.io/v1\nmetadata:\nname: gw1\nspec:\nvpc: test-vpc-1\nsubnet: net1\nlanIp: 10.0.1.254\nselector:\n- \"kubernetes.io/hostname: kube-ovn-worker\"\n- \"kubernetes.io/os: linux\"\nexternalSubnets:\n- ovn-vpc-external-network\n</code></pre> <ul> <li><code>vpc</code>: The VPC to which this VpcNatGateway belongs.</li> <li><code>subnet</code>: A Subnet within the VPC, the VPC Gateway Pod will use <code>lanIp</code> to connect to the tenant network under that subnet.</li> <li><code>lanIp</code>: An unused IP within the <code>subnet</code> that the VPC Gateway Pod will eventually use for the Pod. When configuring routing for a VPC, the  <code>nextHopIP</code> needs to be set to the <code>lanIp</code> of the current VpcNatGateway.</li> <li><code>selector</code>: The node selector for VpcNatGateway Pod has the same format as NodeSelector in Kubernetes.</li> <li><code>externalSubnets</code>: External network used by the VPC gateway, if not configured, <code>ovn-vpc-external-network</code> is used by default, and only one external network is supported in the current version.</li> </ul> <p>Other configurable parameters:</p> <ul> <li><code>tolerations</code>: Configure tolerance for the VPC gateway. For details, see Taints and Tolerations</li> <li><code>affinity</code>: Configure affinity for the Pod or node of the VPC gateway. For details, see Assigning Pods to Nodes</li> </ul>"},{"location":"en/guide/vpc/#create-eip","title":"Create EIP","text":"<p>EIP allows for floating IP, SNAT, and DNAT operations after assigning an IP from an external network segment to a VPC gateway.</p> <p>Randomly assign an address to the EIP:</p> <pre><code>kind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-random\nspec:\nnatGwDp: gw1\n</code></pre> <p>Fixed EIP address assignment:</p> <pre><code>kind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-static\nspec:\nnatGwDp: gw1\nv4ip: 10.0.1.111\n</code></pre> <p>Specify the external network on which the EIP is located:</p> <pre><code>kind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-random\nspec:\nnatGwDp: gw1\nexternalSubnet: ovn-vpc-external-network\n</code></pre> <ul> <li><code>externalSubnet</code>: The name of the external network on which the EIP is located. If not specified, it defaults to <code>ovn-vpc-external-network</code>. If specified, it must be one of the <code>externalSubnets</code> of the VPC gateway.</li> </ul>"},{"location":"en/guide/vpc/#create-dnat-rules","title":"Create DNAT Rules","text":"<p>Through the DNAT rules, external can access to an IP and port within a VPC through an EIP and port.</p> <pre><code>kind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eipd01\nspec:\nnatGwDp: gw1\n\n---\nkind: IptablesDnatRule\napiVersion: kubeovn.io/v1\nmetadata:\nname: dnat01\nspec:\neip: eipd01 externalPort: '8888'\ninternalIp: 10.0.1.10\ninternalPort: '80'\nprotocol: tcp\n</code></pre>"},{"location":"en/guide/vpc/#create-snat-rules","title":"Create SNAT Rules","text":"<p>Through SNAT rules, when a Pod in the VPC accesses an external address, it will go through the corresponding EIP for SNAT.</p> <pre><code>---\nkind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eips01\nspec:\nnatGwDp: gw1\n---\nkind: IptablesSnatRule\napiVersion: kubeovn.io/v1\nmetadata:\nname: snat01\nspec:\neip: eips01\ninternalCIDR: 10.0.1.0/24\n</code></pre>"},{"location":"en/guide/vpc/#create-floating-ip","title":"Create Floating IP","text":"<p>Through floating IP rules, one IP in the VPC will be completely mapped to the EIP, and the external can access the IP in the VPC through this EIP. When the IP in the VPC accesses the external address, it will be SNAT to this EIP</p> <pre><code>---\nkind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eipf01\nspec:\nnatGwDp: gw1\n\n---\nkind: IptablesFIPRule\napiVersion: kubeovn.io/v1\nmetadata:\nname: fip01\nspec:\neip: eipf01\ninternalIp: 10.0.1.5\n</code></pre>"},{"location":"en/guide/vpc/#custom-routing","title":"Custom Routing","text":"<p>Within the custom VPC, users can customize the routing rules within the VPC and combine it with the gateway for more flexible forwarding. Kube-OVN supports static routes and more flexible policy routes.</p>"},{"location":"en/guide/vpc/#static-routes","title":"Static Routes","text":"<pre><code>kind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: test-vpc-1\nspec:\nstaticRoutes:\n- cidr: 0.0.0.0/0\nnextHopIP: 10.0.1.254\npolicy: policyDst\n- cidr: 172.31.0.0/24\nnextHopIP: 10.0.1.253\npolicy: policySrc\nrouteTable: \"rtb1\"\n</code></pre> <ul> <li><code>policy</code>: Supports destination routing <code>policyDst</code> and source routing <code>policySrc</code>.</li> <li>When there are overlapping routing rules, the rule with the longer CIDR mask has higher priority,   and if the mask length is the same, the destination route has a higher priority over the source route.</li> <li><code>routeTable</code>: You can store the route in specific table, default is main table. Associate with subnet please defer to Create Custom Subnets</li> </ul>"},{"location":"en/guide/vpc/#policy-routes","title":"Policy Routes","text":"<p>Traffic matched by static routes can be controlled at a finer granularity by policy routing. Policy routing provides more precise matching rules, priority control and more forwarding actions. This feature brings the OVN internal logical router policy function directly to the outside world, for more information on its use, please refer to Logical Router Policy.</p> <p>An example of policy routes:</p> <pre><code>kind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: test-vpc-1\nspec:\npolicyRoutes:\n- action: drop\nmatch: ip4.src==10.0.1.0/24 &amp;&amp; ip4.dst==10.0.1.250\npriority: 11\n- action: reroute\nmatch: ip4.src==10.0.1.0/24\nnextHopIP: 10.0.1.252\npriority: 10\n</code></pre>"},{"location":"en/guide/vpc/#custom-vpc-dns","title":"Custom vpc-dns","text":"<p>Due to the isolation between custom VPCs and default VPC networks, Pods in VPCs cannot use the default coredns service for domain name resolution. If you want to use coredns to resolve Service domain names within the custom VPC, you can use the <code>vpc-dns</code> resource provided by Kube-OVN.</p>"},{"location":"en/guide/vpc/#create-an-additional-network","title":"Create an Additional Network","text":"<pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: ovn-nad\nnamespace: default\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.0\",\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"ovn-nad.default.ovn\"\n}'\n</code></pre>"},{"location":"en/guide/vpc/#modify-the-provider-of-the-ovn-default-logical-switch","title":"Modify the Provider of the ovn-default Logical Switch","text":"<p>Modify the provider of ovn-default to the provider <code>ovn-nad.default.ovn</code> configured above in nad\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: ovn-default\nspec:\ncidrBlock: 10.16.0.0/16\ndefault: true\ndisableGatewayCheck: false\ndisableInterConnection: false\nenableDHCP: false\nenableIPv6RA: false\nexcludeIps:\n- 10.16.0.1\ngateway: 10.16.0.1\ngatewayType: distributed\nlogicalGateway: false\nnatOutgoing: true\nprivate: false\nprotocol: IPv4\nprovider: ovn-nad.default.ovn\nvpc: ovn-cluster\n</code></pre>"},{"location":"en/guide/vpc/#modify-the-vpc-dns-configmap","title":"Modify the vpc-dns ConfigMap","text":"<p>Create a ConfigMap in the kube-system namespace, configure the vpc-dns parameters to be used for the subsequent vpc-dns feature activation:</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: vpc-dns-config\nnamespace: kube-system\ndata:\ncoredns-vip: 10.96.0.3\nenable-vpc-dns: true\nnad-name: ovn-nad\nnad-provider: ovn-nad.default.ovn\n</code></pre> <ul> <li><code>enable-vpc-dns</code>: (optional) <code>true</code> to enable the feature, <code>false</code> to disable the feature. Default <code>true</code>.</li> <li><code>coredns-image</code>: (optional): DNS deployment image. Default is the cluster coredns deployment version.</li> <li><code>coredns-template</code>: (optional): URL of the DNS deployment template. Default: <code>yamls/coredns-template.yaml</code> in the current version repository.</li> <li><code>coredns-vip</code>: VIP providing LB service for coredns.</li> <li><code>nad-name</code>: Name of the configured <code>network-attachment-definitions</code> resource.</li> <li><code>nad-provider</code>: Name of the used provider.</li> <li><code>k8s-service-host</code>: (optional) IP used by coredns to access the k8s apiserver service.</li> <li><code>k8s-service-port</code>: (optional) Port used by coredns to access the k8s apiserver service.</li> </ul>"},{"location":"en/guide/vpc/#deploying-vpc-dns-dependent-resources","title":"Deploying VPC-DNS Dependent Resources","text":"<pre><code>apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\nlabels:\nkubernetes.io/bootstrapping: rbac-defaults\nname: system:vpc-dns\nrules:\n- apiGroups:\n- \"\"\nresources:\n- endpoints\n- services\n- pods\n- namespaces\nverbs:\n- list\n- watch\n- apiGroups:\n- discovery.k8s.io\nresources:\n- endpointslices\nverbs:\n- list\n- watch\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\nannotations:\nrbac.authorization.kubernetes.io/autoupdate: \"true\"\nlabels:\nkubernetes.io/bootstrapping: rbac-defaults\nname: vpc-dns\nroleRef:\napiGroup: rbac.authorization.k8s.io\nkind: ClusterRole\nname: system:vpc-dns\nsubjects:\n- kind: ServiceAccount\nname: vpc-dns\nnamespace: kube-system\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\nname: vpc-dns\nnamespace: kube-system\n---\napiVersion: v1\nkind: ConfigMap\nmetadata:\nname: vpc-dns-corefile\nnamespace: kube-system\ndata:\nCorefile: |\n.:53 {\nerrors\nhealth {\nlameduck 5s\n}\nready\nkubernetes cluster.local in-addr.arpa ip6.arpa {\npods insecure\nfallthrough in-addr.arpa ip6.arpa\n}\nprometheus :9153\nforward . /etc/resolv.conf {\nprefer_udp\n}\ncache 30\nloop\nreload\nloadbalance\n}\n</code></pre>"},{"location":"en/guide/vpc/#deploy-vpc-dns","title":"Deploy vpc-dns","text":"<pre><code>kind: VpcDns\napiVersion: kubeovn.io/v1\nmetadata:\nname: test-cjh1\nspec:\nvpc: cjh-vpc-1\nsubnet: cjh-subnet-1\n</code></pre> <ul> <li><code>vpc</code>: The VPC name used to deploy the DNS component.</li> <li><code>subnet</code>: The subnet name used to deploy the DNS component.</li> </ul> <p>View resource information:</p> <pre><code>[root@hci-dev-mst-1 kubeovn]# kubectl get vpc-dns\nNAME        ACTIVE   VPC         SUBNET   \ntest-cjh1   false    cjh-vpc-1   cjh-subnet-1   \ntest-cjh2   true     cjh-vpc-1   cjh-subnet-2 </code></pre> <ul> <li><code>ACTIVE</code>: if the custom vpc-dns is ready.</li> </ul>"},{"location":"en/guide/vpc/#restrictions","title":"Restrictions","text":"<ul> <li>Only one custom DNS component will be deployed in one VPC;</li> <li>When multiple VPC-DNS resources (i.e. different subnets in the same VPC) are configured in one VPC, only one VPC-DNS resource with status <code>true</code> will be active, while the others will be <code>false</code>;</li> <li>When the <code>true</code> VPC-DNS is deleted, another <code>false</code> VPC-DNS will be deployed.</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/webhook/","title":"Webhook","text":"<p>Using Webhook, you can verify CRD resources within Kube-OVN. Currently, Webhook mainly performs fixed IP address conflict detection and Subnet CIDR conflict detection, and prompts errors when such conflicts happen.</p> <p>Since Webhook intercepts all Subnet and Pod creation requests, you need to deploy Kube-OVN first and Webhook later.</p>"},{"location":"en/guide/webhook/#install-cert-manager","title":"Install Cert-Manager","text":"<p>Webhook deployment requires certificate, we use cert-manager to generate the associated certificate, we need to deploy cert-manager before deploying Webhook.</p> <p>You can use the following command to deploy cert-manager:</p> <pre><code>kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.8.0/cert-manager.yaml\n</code></pre> <p>More cert-manager usage please refer to cert-manager document\u3002</p>"},{"location":"en/guide/webhook/#install-webhook","title":"Install Webhook","text":"<p>Download Webhook yaml and install:</p> <pre><code># kubectl apply -f https://raw.githubusercontent.com/kubeovn/kube-ovn/master/yamls/webhook.yaml\ndeployment.apps/kube-ovn-webhook created\nservice/kube-ovn-webhook created\nvalidatingwebhookconfiguration.admissionregistration.k8s.io/kube-ovn-webhook created\ncertificate.cert-manager.io/kube-ovn-webhook-serving-cert created\nissuer.cert-manager.io/kube-ovn-webhook-selfsigned-issuer created\n</code></pre>"},{"location":"en/guide/webhook/#verify-webhook-take-effect","title":"Verify Webhook Take Effect","text":"<p>Check the running Pod and get the Pod IP <code>10.16.0.15</code>:</p> <pre><code># kubectl get pod -o wide\nNAME                      READY   STATUS    RESTARTS   AGE     IP           NODE              NOMINATED NODE   READINESS GATES\nstatic-7584848b74-fw9dm   1/1     Running   0          2d13h   10.16.0.15   kube-ovn-worker   &lt;none&gt; </code></pre> <p>Write yaml to create a Pod with the same IP:</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nannotations:\novn.kubernetes.io/ip_address: 10.16.0.15\novn.kubernetes.io/mac_address: 00:00:00:53:6B:B6\nlabels:\napp: static\nmanagedFields:\nname: staticip-pod\nnamespace: default\nspec:\ncontainers:\n- image: docker.io/library/nginx:alpine\nimagePullPolicy: IfNotPresent\nname: qatest\n</code></pre> <p>When using the above yaml to create a fixed address Pod, it prompts an IP address conflict:</p> <pre><code># kubectl apply -f pod-static.yaml\nError from server (annotation ip address 10.16.0.15 is conflict with ip crd static-7584848b74-fw9dm.default 10.16.0.15): error when creating \"pod-static.yaml\": admission webhook \"pod-ip-validaing.kube-ovn.io\" denied the request: annotation ip address 10.16.0.15 is conflict with ip crd static-7584848b74-fw9dm.default 10.16.0.15\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/ops/change-default-subnet/","title":"Change Subnet CIDR","text":"<p>If a subnet CIDR is created that conflicts or does not meet expectations, it can be modified by following the steps in this document.</p> <p>After modifying the subnet CIDR, the previously created Pods will not be able to access the network properly and need to be rebuilt. Careful consideration is recommended before operating\u3002This document is only for business subnet CIDR changes, if you need to Change the Join subnet CIDR, please refer to Change Join CIDR.</p>"},{"location":"en/ops/change-default-subnet/#edit-subnet","title":"Edit Subnet","text":"<p>Use <code>kubectl edit</code> to modify <code>cidrBlock</code>\uff0c<code>gateway</code> and <code>excludeIps</code>.</p> <pre><code>kubectl edit subnet test-subnet\n</code></pre>"},{"location":"en/ops/change-default-subnet/#rebuild-all-pods-under-this-subnet","title":"Rebuild all Pods under this Subnet","text":"<p>Take the subnet binding <code>test</code> Namespace as example:</p> <pre><code>for pod in $(kubectl get pod --no-headers -n \"$ns\" --field-selector spec.restartPolicy=Always -o custom-columns=NAME:.metadata.name,HOST:spec.hostNetwork | awk '{if ($2!=\"true\") print $1}'); do\nkubectl delete pod \"$pod\" -n test --ignore-not-found\ndone\n</code></pre> <p>If only the default subnet is used, you can delete all Pods that are not in host network mode using the following command:</p> <pre><code>for ns in $(kubectl get ns --no-headers -o custom-columns=NAME:.metadata.name); do\nfor pod in $(kubectl get pod --no-headers -n \"$ns\" --field-selector spec.restartPolicy=Always -o custom-columns=NAME:.metadata.name,HOST:spec.hostNetwork | awk '{if ($2!=\"true\") print $1}'); do\nkubectl delete pod \"$pod\" -n \"$ns\" --ignore-not-found\n  done\ndone\n</code></pre>"},{"location":"en/ops/change-default-subnet/#change-default-subnet-settings","title":"Change Default Subnet Settings","text":"<p>If you are modifying the CIDR for the default Subnet, you also need to change the args of the <code>kube-ovn-controller</code> Deployment:</p> <pre><code>args:\n- --default-cidr=10.17.0.0/16\n- --default-gateway=10.17.0.1\n- --default-exclude-ips=10.17.0.1\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/ops/change-join-subnet/","title":"Change Join Subnet CIDR","text":"<p>If the Join subnet CIDR created conflicts or does not meet expectations, you can use this document to modify.</p> <p>After modifying the Join Subnet CIDR, the previously created Pods will not be able to access the external network normally and need to wait for the rebuild completed.</p>"},{"location":"en/ops/change-join-subnet/#delete-join-subnet","title":"Delete Join Subnet","text":"<pre><code>kubectl patch subnet join --type='json' -p '[{\"op\": \"replace\", \"path\": \"/metadata/finalizers\", \"value\": []}]'\nkubectl delete subnet join\n</code></pre>"},{"location":"en/ops/change-join-subnet/#cleanup-allocated-config","title":"Cleanup Allocated Config","text":"<pre><code>kubectl annotate node ovn.kubernetes.io/allocated=false --all --overwrite\n</code></pre>"},{"location":"en/ops/change-join-subnet/#modify-join-subnet","title":"Modify Join Subnet","text":"<p>Change Join Subnet args in <code>kube-ovn-controller</code>:</p> <pre><code>kubectl edit deployment -n kube-system kube-ovn-controller\n</code></pre> <p>Change the CIDR below:</p> <pre><code>args:\n- --node-switch-cidr=100.51.0.0/16\n</code></pre> <p>Reboot the <code>kube-ovn-controller</code> and rebuild <code>join</code> Subnet:</p> <pre><code>kubectl delete pod -n kube-system -lapp=kube-ovn-controller\n</code></pre> <p>Check the new Join Subnet information:</p> <pre><code># kubectl get subnet\nNAME          PROVIDER   VPC           PROTOCOL   CIDR            PRIVATE   NAT     DEFAULT   GATEWAYTYPE   V4USED   V4AVAILABLE   V6USED   V6AVAILABLE   EXCLUDEIPS\njoin          ovn        ovn-cluster   IPv4       100.51.0.0/16   false     false   false     distributed   2        65531         0        0             [\"100.51.0.1\"]\novn-default   ovn        ovn-cluster   IPv4       10.17.0.0/16    false     true    true      distributed   5        65528         0        0             [\"10.17.0.1\"]\n</code></pre>"},{"location":"en/ops/change-join-subnet/#reconfigure-ovn0-nic-address","title":"Reconfigure ovn0 NIC Address","text":"<p>The <code>ovn0</code> NIC information for each node needs to be re-updated, which can be done by restarting <code>kube-ovn-cni</code>:</p> <pre><code>kubectl delete pod -n kube-system -l app=kube-ovn-cni\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/ops/change-log-level/","title":"Change Log Level","text":"<p>Open <code>kube-ovn.yaml</code> and set the log level in the parameter list of the service startup script, such as:</p> <pre><code>vi kube-ovn.yaml\n# ...\n- name: kube-ovn-controller\n          image: \"docker.io/kubeovn/kube-ovn:v1.13.0\"\nimagePullPolicy: IfNotPresent\n          args:\n          - /kube-ovn/start-controller.sh\n          - --v=3\n# ...\n# The higher the log level, the more detailed the log\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/ops/change-ovn-central-node/","title":"Replace ovn-central Node","text":"<p>Since <code>ovn-nb</code> and <code>ovn-sb</code> within <code>ovn-central</code> create separate etcd-like raft clusters, replacing the <code>ovn-central</code> node requires additional operations to ensure correct cluster state and consistent data. It is recommended that only one node be up and down at a time to avoid the cluster going into an unavailable state and affecting the overall cluster network.</p>"},{"location":"en/ops/change-ovn-central-node/#ovn-central-nodes-offline","title":"ovn-central Nodes Offline","text":"<p>This document use the cluster below to describes how to remove the <code>kube-ovn-control-plane2</code> node from the <code>ovn-central</code> as an example.</p> <pre><code># kubectl -n kube-system get pod -o wide | grep central\novn-central-6bf58cbc97-2cdhg                      1/1     Running   0             21m   172.18.0.3   kube-ovn-control-plane    &lt;none&gt;           &lt;none&gt;\novn-central-6bf58cbc97-crmfp                      1/1     Running   0             21m   172.18.0.5   kube-ovn-control-plane2   &lt;none&gt;           &lt;none&gt;\novn-central-6bf58cbc97-lxmpl                      1/1     Running   0             21m   172.18.0.4   kube-ovn-control-plane3   &lt;none&gt;           &lt;none&gt;\n</code></pre>"},{"location":"en/ops/change-ovn-central-node/#kick-node-in-ovn-nb","title":"Kick Node in ovn-nb","text":"<p>First check the ID of the node within the cluster for subsequent operations.</p> <pre><code># kubectl ko nb status\n1b9a\nName: OVN_Northbound\nCluster ID: 32ca (32ca07fb-739b-4257-b510-12fa18e7cce8)\nServer ID: 1b9a (1b9a5d76-e69b-410c-8085-39943d0cd38c)\nAddress: tcp:[172.18.0.3]:6643\nStatus: cluster member\nRole: leader\nTerm: 1\nLeader: self\nVote: self\n\nLast Election started 2135194 ms ago, reason: timeout\nLast Election won: 2135188 ms ago\nElection timer: 5000\nLog: [135, 135]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: &lt;-d64b -&gt;d64b &lt;-4984 -&gt;4984\nDisconnections: 0\nServers:\n    4984 (4984 at tcp:[172.18.0.4]:6643) next_index=135 match_index=134 last msg 1084 ms ago\n    1b9a (1b9a at tcp:[172.18.0.3]:6643) (self) next_index=2 match_index=134\nd64b (d64b at tcp:[172.18.0.5]:6643) next_index=135 match_index=134 last msg 1084 ms ago\nstatus: ok\n</code></pre> <p><code>kube-ovn-control-plane2</code> corresponds to a node IP of <code>172.18.0.5</code> and the corresponding ID within the cluster is <code>d64b</code>. Next, kick the node out of the ovn-nb cluster.</p> <pre><code># kubectl ko nb kick d64b\nstarted removal\n</code></pre> <p>Check if the node has been kicked:</p> <pre><code># kubectl ko nb status\n1b9a\nName: OVN_Northbound\nCluster ID: 32ca (32ca07fb-739b-4257-b510-12fa18e7cce8)\nServer ID: 1b9a (1b9a5d76-e69b-410c-8085-39943d0cd38c)\nAddress: tcp:[172.18.0.3]:6643\nStatus: cluster member\nRole: leader\nTerm: 1\nLeader: self\nVote: self\n\nLast Election started 2297649 ms ago, reason: timeout\nLast Election won: 2297643 ms ago\nElection timer: 5000\nLog: [136, 136]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: &lt;-4984 -&gt;4984\nDisconnections: 2\nServers:\n    4984 (4984 at tcp:[172.18.0.4]:6643) next_index=136 match_index=135 last msg 1270 ms ago\n    1b9a (1b9a at tcp:[172.18.0.3]:6643) (self) next_index=2 match_index=135\nstatus: ok\n</code></pre>"},{"location":"en/ops/change-ovn-central-node/#kick-node-in-ovn-sb","title":"Kick Node in ovn-sb","text":"<p>Next, for the ovn-sb cluster, you need to first check the ID of the node within the cluster for subsequent operations.</p> <pre><code>kubectl ko sb status\n3722\nName: OVN_Southbound\nCluster ID: d4bd (d4bd37a4-0400-499f-b4df-b4fd389780f0)\nServer ID: 3722 (3722d5ae-2ced-4820-a6b2-8b744d11fb3e)\nAddress: tcp:[172.18.0.3]:6644\nStatus: cluster member\nRole: leader\nTerm: 1\nLeader: self\nVote: self\n\nLast Election started 2395317 ms ago, reason: timeout\nLast Election won: 2395316 ms ago\nElection timer: 5000\nLog: [130, 130]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: &lt;-e9f7 -&gt;e9f7 &lt;-6e84 -&gt;6e84\nDisconnections: 0\nServers:\n    e9f7 (e9f7 at tcp:[172.18.0.5]:6644) next_index=130 match_index=129 last msg 1006 ms ago\n    6e84 (6e84 at tcp:[172.18.0.4]:6644) next_index=130 match_index=129 last msg 1004 ms ago\n    3722 (3722 at tcp:[172.18.0.3]:6644) (self) next_index=2 match_index=129\nstatus: ok\n</code></pre> <p><code>kube-ovn-control-plane2</code> corresponds to node IP <code>172.18.0.5</code> and the corresponding ID within the cluster is <code>e9f7</code>. Next, kick the node out of the ovn-sb cluster.</p> <pre><code># kubectl ko sb kick e9f7\nstarted removal\n</code></pre> <p>Check if the node has been kicked:</p> <pre><code># kubectl ko sb status\n3722\nName: OVN_Southbound\nCluster ID: d4bd (d4bd37a4-0400-499f-b4df-b4fd389780f0)\nServer ID: 3722 (3722d5ae-2ced-4820-a6b2-8b744d11fb3e)\nAddress: tcp:[172.18.0.3]:6644\nStatus: cluster member\nRole: leader\nTerm: 1\nLeader: self\nVote: self\n\nLast Election started 2481636 ms ago, reason: timeout\nLast Election won: 2481635 ms ago\nElection timer: 5000\nLog: [131, 131]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: &lt;-6e84 -&gt;6e84\nDisconnections: 2\nServers:\n    6e84 (6e84 at tcp:[172.18.0.4]:6644) next_index=131 match_index=130 last msg 642 ms ago\n    3722 (3722 at tcp:[172.18.0.3]:6644) (self) next_index=2 match_index=130\nstatus: ok\n</code></pre>"},{"location":"en/ops/change-ovn-central-node/#delete-node-label-and-downscale-ovn-central","title":"Delete Node Label and Downscale ovn-central","text":"<p>Note that you need to remove the offline node from the node address of the ovn-central environment variable <code>NODE_IPS</code>.</p> <pre><code>kubectl label node kube-ovn-control-plane2 kube-ovn/role-\nkubectl scale deployment -n kube-system ovn-central --replicas=2\nkubectl set env deployment/ovn-central -n kube-system NODE_IPS=\"172.18.0.3,172.18.0.4\"\nkubectl rollout status deployment/ovn-central -n kube-system </code></pre>"},{"location":"en/ops/change-ovn-central-node/#modify-components-address-to-ovn-central","title":"Modify Components Address to ovn-central","text":"<p>Modify <code>ovs-ovn</code> to remove the offline Node address:</p> <pre><code># kubectl set env daemonset/ovs-ovn -n kube-system OVN_DB_IPS=\"172.18.0.3,172.18.0.4\"\ndaemonset.apps/ovs-ovn env updated\n# kubectl delete pod -n kube-system -lapp=ovs\npod \"ovs-ovn-4f6jc\" deleted\npod \"ovs-ovn-csn2w\" deleted\npod \"ovs-ovn-mpbmb\" deleted\n</code></pre> <p>Modify <code>kube-ovn-controller</code> to remove the offline Node address:</p> <pre><code># kubectl set env deployment/kube-ovn-controller -n kube-system OVN_DB_IPS=\"172.18.0.3,172.18.0.4\"\ndeployment.apps/kube-ovn-controller env updated\n\n# kubectl rollout status deployment/kube-ovn-controller -n kube-system\nWaiting for deployment \"kube-ovn-controller\" rollout to finish: 1 of 3 updated replicas are available...\nWaiting for deployment \"kube-ovn-controller\" rollout to finish: 2 of 3 updated replicas are available...\ndeployment \"kube-ovn-controller\" successfully rolled out\n</code></pre>"},{"location":"en/ops/change-ovn-central-node/#clean-node","title":"Clean Node","text":"<p>Delete the database files in the <code>kube-ovn-control-plane2</code> node to avoid errors when adding the node again:</p> <pre><code>rm -rf /etc/origin/ovn\n</code></pre> <p>To take a node offline from a Kubernetes cluster entirely, please continue with Delete Work Node.</p>"},{"location":"en/ops/change-ovn-central-node/#ovn-central-online","title":"ovn-central Online","text":"<p>The following steps will add a new Kubernetes node to the <code>ovn-central</code> cluster.</p>"},{"location":"en/ops/change-ovn-central-node/#directory-check","title":"Directory Check","text":"<p>Check if the <code>ovnnb_db.db</code> or <code>ovnsb_db.db</code> file exists in the <code>/etc/origin/ovn</code> directory of the new node, and if so, delete it:</p> <pre><code>rm -rf /etc/origin/ovn\n</code></pre>"},{"location":"en/ops/change-ovn-central-node/#check-current-ovn-central-status","title":"Check Current ovn-central Status","text":"<p>If the current <code>ovn-central</code> cluster state is already abnormal, adding new nodes may cause the voting election to fail to pass the majority, affecting subsequent operations.</p> <pre><code># kubectl ko nb status\n1b9a\nName: OVN_Northbound\nCluster ID: 32ca (32ca07fb-739b-4257-b510-12fa18e7cce8)\nServer ID: 1b9a (1b9a5d76-e69b-410c-8085-39943d0cd38c)\nAddress: tcp:[172.18.0.3]:6643\nStatus: cluster member\nRole: leader\nTerm: 44\nLeader: self\nVote: self\n\nLast Election started 1855739 ms ago, reason: timeout\nLast Election won: 1855729 ms ago\nElection timer: 5000\nLog: [147, 147]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: -&gt;4984 &lt;-4984\nDisconnections: 0\nServers:\n    4984 (4984 at tcp:[172.18.0.4]:6643) next_index=147 match_index=146 last msg 367 ms ago\n    1b9a (1b9a at tcp:[172.18.0.3]:6643) (self) next_index=140 match_index=146\nstatus: ok\n\n# kubectl ko sb status\n3722\nName: OVN_Southbound\nCluster ID: d4bd (d4bd37a4-0400-499f-b4df-b4fd389780f0)\nServer ID: 3722 (3722d5ae-2ced-4820-a6b2-8b744d11fb3e)\nAddress: tcp:[172.18.0.3]:6644\nStatus: cluster member\nRole: leader\nTerm: 33\nLeader: self\nVote: self\n\nLast Election started 1868589 ms ago, reason: timeout\nLast Election won: 1868579 ms ago\nElection timer: 5000\nLog: [142, 142]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: -&gt;6e84 &lt;-6e84\nDisconnections: 0\nServers:\n    6e84 (6e84 at tcp:[172.18.0.4]:6644) next_index=142 match_index=141 last msg 728 ms ago\n    3722 (3722 at tcp:[172.18.0.3]:6644) (self) next_index=134 match_index=141\nstatus: ok\n</code></pre>"},{"location":"en/ops/change-ovn-central-node/#label-node-and-scale-ovn-central","title":"Label Node and Scale ovn-central","text":"<p>Note that you need to add the online node address to the node address of the ovn-central environment variable <code>NODE_IPS</code>.</p> <pre><code>kubectl label node kube-ovn-control-plane2 kube-ovn/role=master\nkubectl scale deployment -n kube-system ovn-central --replicas=3\nkubectl set env deployment/ovn-central -n kube-system NODE_IPS=\"172.18.0.3,172.18.0.4,172.18.0.5\"\nkubectl rollout status deployment/ovn-central -n kube-system\n</code></pre>"},{"location":"en/ops/change-ovn-central-node/#modify-components-address-to-ovn-central_1","title":"Modify Components Address to ovn-central","text":"<p>Modify <code>ovs-ovn</code> to add the online Node address:</p> <pre><code># kubectl set env daemonset/ovs-ovn -n kube-system OVN_DB_IPS=\"172.18.0.3,172.18.0.4,172.18.0.5\"\ndaemonset.apps/ovs-ovn env updated\n# kubectl delete pod -n kube-system -lapp=ovs\npod \"ovs-ovn-4f6jc\" deleted\npod \"ovs-ovn-csn2w\" deleted\npod \"ovs-ovn-mpbmb\" deleted\n</code></pre> <p>Modify <code>kube-ovn-controller</code> to add the online Node address:</p> <pre><code># kubectl set env deployment/kube-ovn-controller -n kube-system OVN_DB_IPS=\"172.18.0.3,172.18.0.4,172.18.0.5\"\ndeployment.apps/kube-ovn-controller env updated\n\n# kubectl rollout status deployment/kube-ovn-controller -n kube-system\nWaiting for deployment \"kube-ovn-controller\" rollout to finish: 1 of 3 updated replicas are available...\nWaiting for deployment \"kube-ovn-controller\" rollout to finish: 2 of 3 updated replicas are available...\ndeployment \"kube-ovn-controller\" successfully rolled out\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/ops/delete-worker-node/","title":"Delete Work Node","text":"<p>If the node is simply removed from Kubernetes, the <code>ovn-controller</code> process running in <code>ovs-ovn</code> on the node will periodically connect to <code>ovn-central</code> to register relevant network information. This leads to additional resource waste and potential rule conflict risk\u3002 Therefore, when removing nodes from within Kubernetes, follow the steps below to ensure that related resources are cleaned up properly.</p> <p>This document describes the steps to delete a worker node, if you want to change the node where <code>ovn-central</code> is located, please refer to Replace ovn-central Node.</p>"},{"location":"en/ops/delete-worker-node/#evict-pods-on-the-node","title":"Evict Pods on the Node","text":"<pre><code> # kubectl drain kube-ovn-worker --ignore-daemonsets --force\nnode/kube-ovn-worker cordoned\n WARNING: ignoring DaemonSet-managed Pods: kube-system/kube-ovn-cni-zt74b, kube-system/kube-ovn-pinger-5rxfs, kube-system/kube-proxy-jpmnm, kube-system/ovs-ovn-v2kll\n evicting pod kube-system/coredns-64897985d-qsgpt\n evicting pod local-path-storage/local-path-provisioner-5ddd94ff66-llss6\n evicting pod kube-system/kube-ovn-controller-8459db5ff4-94lxb\n pod/kube-ovn-controller-8459db5ff4-94lxb evicted\n pod/coredns-64897985d-qsgpt evicted\n pod/local-path-provisioner-5ddd94ff66-llss6 evicted\n node/kube-ovn-worker drained\n</code></pre>"},{"location":"en/ops/delete-worker-node/#stop-kubelet-and-docker","title":"Stop kubelet and docker","text":"<p>This step stops the <code>ovs-ovn</code> container to avoid registering information to <code>ovn-central</code>. Log into to the corresponding node and ruu the following commands:</p> <pre><code>systemctl stop kubelet\nsystemctl stop docker\n</code></pre> <p>If using containerd as the CRI, the following command needs to be executed to stop the <code>ovs-ovn</code> container:</p> <pre><code>crictl rm -f $(crictl ps | grep openvswitch | awk '{print $1}')\n</code></pre>"},{"location":"en/ops/delete-worker-node/#cleanup-files-on-node","title":"Cleanup Files on Node","text":"<pre><code>rm -rf /var/run/openvswitch\nrm -rf /var/run/ovn\nrm -rf /etc/origin/openvswitch/\nrm -rf /etc/origin/ovn/\nrm -rf /etc/cni/net.d/00-kube-ovn.conflist\nrm -rf /etc/cni/net.d/01-kube-ovn.conflist\nrm -rf /var/log/openvswitch\nrm -rf /var/log/ovn\n</code></pre>"},{"location":"en/ops/delete-worker-node/#delete-the-node","title":"Delete the Node","text":"<pre><code>kubectl delete no kube-ovn-01\n</code></pre>"},{"location":"en/ops/delete-worker-node/#check-if-node-removed-from-ovn-sb","title":"Check If Node Removed from OVN-SB","text":"<p>In the example below, the node <code>kube-ovn-worker</code> is not removed:</p> <pre><code># kubectl ko sbctl show\nChassis \"b0564934-5a0d-4804-a4c0-476c93596a17\"\nhostname: kube-ovn-worker\n  Encap geneve\n      ip: \"172.18.0.2\"\noptions: {csum=\"true\"}\nPort_Binding kube-ovn-pinger-5rxfs.kube-system\nChassis \"6a29de7e-d731-4eaf-bacd-2f239ee52b28\"\nhostname: kube-ovn-control-plane\n  Encap geneve\n      ip: \"172.18.0.3\"\noptions: {csum=\"true\"}\nPort_Binding coredns-64897985d-nbfln.kube-system\n  Port_Binding node-kube-ovn-control-plane\n  Port_Binding local-path-provisioner-5ddd94ff66-h4tn9.local-path-storage\n  Port_Binding kube-ovn-pinger-hf2p6.kube-system\n  Port_Binding coredns-64897985d-fhwlw.kube-system\n</code></pre>"},{"location":"en/ops/delete-worker-node/#delete-the-chassis-manually","title":"Delete the Chassis Manually","text":"<p>Use the uuid find above to delete the chassis:</p> <pre><code># kubectl ko sbctl chassis-del b0564934-5a0d-4804-a4c0-476c93596a17\n# kubectl ko sbctl show\nChassis \"6a29de7e-d731-4eaf-bacd-2f239ee52b28\"\nhostname: kube-ovn-control-plane\n  Encap geneve\n      ip: \"172.18.0.3\"\noptions: {csum=\"true\"}\nPort_Binding coredns-64897985d-nbfln.kube-system\n  Port_Binding node-kube-ovn-control-plane\n  Port_Binding local-path-provisioner-5ddd94ff66-h4tn9.local-path-storage\n  Port_Binding kube-ovn-pinger-hf2p6.kube-system\n  Port_Binding coredns-64897985d-fhwlw.kube-system\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/ops/faq/","title":"FAQ","text":""},{"location":"en/ops/faq/#kylin-arm-system-cross-host-container-access-intermittently-fails","title":"Kylin ARM system cross-host container access intermittently fails","text":""},{"location":"en/ops/faq/#behavior","title":"Behavior","text":"<p>There is a problem with Kylin ARM system and some NIC offload, which can cause intermittent container network failure.</p> <p>Use <code>netstat</code> to identify the problem:</p> <pre><code># netstat -us\nIcmpMsg:\n    InType0: 22\nInType3: 24\nInType8: 117852\nOutType0: 117852\nOutType3: 29\nOutType8: 22\nUdp:\n    3040636 packets received\n    0 packets to unknown port received.\n    4 packet receive errors\n    602 packets sent\n    0 receive buffer errors\n    0 send buffer errors\n    InCsumErrors: 4\nUdpLite:\nIpExt:\n    InBcastPkts: 10244\nInOctets: 4446320361\nOutOctets: 1496815600\nInBcastOctets: 3095950\nInNoECTPkts: 7683903\n</code></pre> <p>If <code>InCsumErrors</code> is present and increases with netwoork failures, you can confirm that this is the problem.</p>"},{"location":"en/ops/faq/#solution","title":"Solution","text":"<p>The fundamental solution requires communication with Kylin and the corresponding network card manufacturer to update the system and drivers. A temporary solution would be to turn off <code>tx offload</code> on the physical NIC, but this would cause a significant degradation in tcp performance.</p> <pre><code>ethtool -K eth0 tx off\n</code></pre> <p>From the community feedback, the problem can be solved by the <code>4.19.90-25.16.v2101</code> kernel.</p>"},{"location":"en/ops/faq/#pod-can-not-access-service","title":"Pod can not Access Service","text":""},{"location":"en/ops/faq/#behavior_1","title":"Behavior","text":"<p>Pod can not access Service, and <code>dmesg</code> show errors:</p> <pre><code>netlink\uff1aUnknown conntrack attr (type=6, max=5)\nopenvswitch: netlink: Flow actions may not be safe on all matching packets.\n</code></pre> <p>This log indicates that the in-kernel OVS version is too low to support the corresponding NAT operation.</p>"},{"location":"en/ops/faq/#solution_1","title":"Solution","text":"<ol> <li>Upgrade the kernel module or compile the OVS kernel module manually.</li> <li>If you are using an Overlay network you can change the <code>kube-ovn-controller</code> args, setting <code>--enable-lb=false</code> to disable the OVN LB to use kube-proxy for service forwarding.</li> </ol> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/ops/from-calico/","title":"Install Kube-OVN From Calico","text":"<p>If a Kubernetes cluster already has Calico installed and needs to change to Kube-OVN you can refer to this document.</p> <p>Since the installation of Calico may vary from version to version and the existing Pod network may be disrupted during the replacement process, it is recommended that you plan ahead and compare the differences in Calico installation from version to version.</p>"},{"location":"en/ops/from-calico/#uninstall-calico","title":"Uninstall Calico","text":"<p>For Calico installed from an Operator:</p> <pre><code>kubectl delete -f https://projectcalico.docs.tigera.io/manifests/tigera-operator.yaml\nkubectl delete -f https://projectcalico.docs.tigera.io/manifests/custom-resources.yaml\n</code></pre> <p>For Calico installed from manifests:</p> <pre><code>kubectl delete -f https://projectcalico.docs.tigera.io/manifests/calico.yaml </code></pre>"},{"location":"en/ops/from-calico/#cleanup-config-files","title":"Cleanup Config Files","text":"<p>Delete the CNI-related configuration files on each machine, depending on the environment:</p> <pre><code>rm -f /etc/cni/net.d/10-calico.conflist\nrm -f /etc/cni/net.d/calico-kubeconfig\n</code></pre> <p>Calico still leaves routing rules, iptables rules, veth network interfaces and other configuration information on the node, so it is recommended to reboot the node to clean up the relevant configuration to avoid problems that are difficult to troubleshoot.</p>"},{"location":"en/ops/from-calico/#install-kube-ovn","title":"Install Kube-OVN","text":"<p>You can refer to One Click Installation to install Kube-OVN as usual.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/ops/kubectl-ko/","title":"Kubectl Plugin","text":"<p>To facilitate daily operations and maintenance, Kube-OVN provides the kubectl plug-in tool, which allows administrators to perform daily operations through this command. For examples: Check OVN database information and status, OVN database backup and restore, OVS related information, tcpdump specific containers, specific link logical topology, network problem diagnosis and performance optimization.</p>"},{"location":"en/ops/kubectl-ko/#plugin-installation","title":"Plugin Installation","text":"<p>Kube-OVN installation will deploy the plugin to each node by default. If the machine that runs kubectl is not in the cluster, or if you need to reinstall the plugin, please refer to the following steps:</p> <p>Download <code>kubectl-ko</code> file:</p> <pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/images/kubectl-ko\n</code></pre> <p>Move file to <code>$PATH</code>:</p> <pre><code>mv kubectl-ko /usr/local/bin/kubectl-ko\n</code></pre> <p>Add executable permissions:</p> <pre><code>chmod +x /usr/local/bin/kubectl-ko\n</code></pre> <p>Check if the plugin works properly:</p> <pre><code># kubectl plugin list\nThe following compatible plugins are available:\n\n/usr/local/bin/kubectl-ko\n</code></pre>"},{"location":"en/ops/kubectl-ko/#plugin-usage","title":"Plugin Usage","text":"<p>Running <code>kubectl ko</code> will show all the available commands and usage descriptions, as follows:</p> <pre><code># kubectl ko\nkubectl ko {subcommand} [option...]\nAvailable Subcommands:\n  [nb|sb] [status|kick|backup|dbstatus|restore]     ovn-db operations show cluster status, kick stale server, backup database, get db consistency status or restore ovn nb db when met 'inconsistent data' error\n  nbctl [ovn-nbctl options ...]    invoke ovn-nbctl\n  sbctl [ovn-sbctl options ...]    invoke ovn-sbctl\n  vsctl {nodeName} [ovs-vsctl options ...]   invoke ovs-vsctl on the specified node\n  ofctl {nodeName} [ovs-ofctl options ...]   invoke ovs-ofctl on the specified node\n  dpctl {nodeName} [ovs-dpctl options ...]   invoke ovs-dpctl on the specified node\n  appctl {nodeName} [ovs-appctl options ...]   invoke ovs-appctl on the specified node\n  tcpdump {namespace/podname} [tcpdump options ...]     capture pod traffic\n  trace {namespace/podname} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp/udp port]    trace ICMP/TCP/UDP\n  trace {namespace/podname} {target ip address} [target mac address] arp {request|reply}                     trace ARP request/reply\n  trace {node//nodename} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp/udp port]       trace ICMP/TCP/UDP\n  trace {node//nodename} {target ip address} [target mac address] arp {request|reply}                        trace ARP request/reply\n  echo \"  diagnose {all|node|subnet|IPPorts} [nodename|subnetName|{proto1}-{IP1}-{Port1},{proto2}-{IP2}-{Port2}]    diagnose connectivity of all nodes or a specific node or specify subnet's ds pod or IPPorts like 'tcp-172.18.0.2-53,udp-172.18.0.3-53'\"\ntuning {install-fastpath|local-install-fastpath|remove-fastpath|install-stt|local-install-stt|remove-stt} {centos7|centos8}} [kernel-devel-version]  deploy  kernel optimisation components to the system\n  reload    restart all kube-ovn components\n  log {kube-ovn|ovn|ovs|linux|all}    save log to ./kubectl-ko-log/\n  perf [image] performance test default image is kubeovn/test:v1.12.0  </code></pre> <p>The specific functions and usage of each command are described below.</p>"},{"location":"en/ops/kubectl-ko/#nb-sb-status-kick-backup-dbstatus-restore","title":"[nb | sb] [status | kick | backup | dbstatus | restore]","text":"<p>This subcommand mainly operates on OVN northbound or southbound databases, including database cluster status check, database node offline, database backup, database storage status check and database repair.</p>"},{"location":"en/ops/kubectl-ko/#db-cluster-status-check","title":"DB Cluster Status Check","text":"<p>This command executes <code>ovs-appctl cluster/status</code> on the leader node of the corresponding OVN database to show the cluster status:</p> <pre><code># kubectl ko nb status\n306b\nName: OVN_Northbound\nCluster ID: 9a87 (9a872522-3e7d-47ca-83a3-d74333e1a7ca)\nServer ID: 306b (306b256b-b5e1-4eb0-be91-4ca96adf6bad)\nAddress: tcp:[172.18.0.2]:6643\nStatus: cluster member\nRole: leader\nTerm: 1\nLeader: self\nVote: self\n\nLast Election started 280309 ms ago, reason: timeout\nLast Election won: 280309 ms ago\nElection timer: 5000\nLog: [139, 139]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: &lt;-8723 -&gt;8723 &lt;-85d6 -&gt;85d6\nDisconnections: 0\nServers:\n    85d6 (85d6 at tcp:[172.18.0.4]:6643) next_index=139 match_index=138 last msg 763 ms ago\n    8723 (8723 at tcp:[172.18.0.3]:6643) next_index=139 match_index=138 last msg 763 ms ago\n    306b (306b at tcp:[172.18.0.2]:6643) (self) next_index=2 match_index=138\nstatus: ok\n</code></pre> <p>If the <code>match_index</code> under <code>Server</code> has a large difference and the <code>last msg</code> time is long, the corresponding Server may not respond for a long time and needs to be checked further.</p>"},{"location":"en/ops/kubectl-ko/#db-nodes-offline","title":"DB Nodes Offline","text":"<p>This command removes a node from the OVN database and is required when a node is taken offline or replaced. The following is an example of the cluster status from the previous command, to offline the <code>172.18.0.3</code> node:</p> <pre><code># kubectl ko nb kick 8723\nstarted removal\n</code></pre> <p>Check the database cluster status again to confirm that the node has been removed:</p> <pre><code># kubectl ko nb status\n306b\nName: OVN_Northbound\nCluster ID: 9a87 (9a872522-3e7d-47ca-83a3-d74333e1a7ca)\nServer ID: 306b (306b256b-b5e1-4eb0-be91-4ca96adf6bad)\nAddress: tcp:[172.18.0.2]:6643\nStatus: cluster member\nRole: leader\nTerm: 1\nLeader: self\nVote: self\n\nLast Election started 324356 ms ago, reason: timeout\nLast Election won: 324356 ms ago\nElection timer: 5000\nLog: [140, 140]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: &lt;-85d6 -&gt;85d6\nDisconnections: 2\nServers:\n    85d6 (85d6 at tcp:[172.18.0.4]:6643) next_index=140 match_index=139 last msg 848 ms ago\n    306b (306b at tcp:[172.18.0.2]:6643) (self) next_index=2 match_index=139\nstatus: ok\n</code></pre>"},{"location":"en/ops/kubectl-ko/#db-backup","title":"DB Backup","text":"<p>This subcommand backs up the current OVN database locally and can be used for disaster recovery:</p> <pre><code># kubectl ko nb backup\ntar: Removing leading `/' from member names\nbackup ovn-nb db to /root/ovnnb_db.060223191654183154.backup\n</code></pre>"},{"location":"en/ops/kubectl-ko/#database-storage-status-check","title":"Database Storage Status Check","text":"<p>This command is used to check if the database file is corrupt:</p> <pre><code># kubectl ko nb dbstatus\nstatus: ok\n</code></pre> <p>If error happens, <code>inconsistent data</code> is displayed and needs to be fixed with the following command.</p>"},{"location":"en/ops/kubectl-ko/#database-repair","title":"Database Repair","text":"<p>If the database status goes to <code>inconsistent data</code>, this command can be used to repair:</p> <pre><code># kubectl ko nb restore\ndeployment.apps/ovn-central scaled\novn-central original replicas is 3\nfirst nodeIP is 172.18.0.5\novs-ovn pod on node 172.18.0.5 is ovs-ovn-8jxv9\novs-ovn pod on node 172.18.0.3 is ovs-ovn-sjzb6\novs-ovn pod on node 172.18.0.4 is ovs-ovn-t87zk\nbackup nb db file\nrestore nb db file, operate in pod ovs-ovn-8jxv9\ndeployment.apps/ovn-central scaled\nfinish restore nb db file and ovn-central replicas\nrecreate ovs-ovn pods\npod \"ovs-ovn-8jxv9\" deleted\npod \"ovs-ovn-sjzb6\" deleted\npod \"ovs-ovn-t87zk\" deleted\n</code></pre>"},{"location":"en/ops/kubectl-ko/#nbctl-sbctl-options","title":"[nbctl | sbctl] [options ...]","text":"<p>This subcommand executes the <code>ovn-nbctl</code> and <code>ovn-sbctl</code> commands directly into the leader node of the OVN northbound or southbound database. For more detailed usage of this command, please refer to the official documentation of the upstream OVN ovn-nbctl(8) \u548c ovn-sbctl(8)\u3002</p> <pre><code># kubectl ko nbctl show\nswitch c7cd17e8-ceee-4a91-9bb3-e5a313fe1ece (snat)\nport snat-ovn-cluster\n        type: router\n        router-port: ovn-cluster-snat\nswitch 20e0c6d0-023a-4756-aec5-200e0c60f95d (join)\nport node-liumengxin-ovn3-192.168.137.178\n        addresses: [\"00:00:00:64:FF:A8 100.64.0.4\"]\nport node-liumengxin-ovn1-192.168.137.176\n        addresses: [\"00:00:00:AF:98:62 100.64.0.2\"]\nport node-liumengxin-ovn2-192.168.137.177\n        addresses: [\"00:00:00:D9:58:B8 100.64.0.3\"]\nport join-ovn-cluster\n        type: router\n        router-port: ovn-cluster-join\nswitch 0191705c-f827-427b-9de3-3c3b7d971ba5 (central)\nport central-ovn-cluster\n        type: router\n        router-port: ovn-cluster-central\nswitch 2a45ff05-388d-4f85-9daf-e6fccd5833dc (ovn-default)\nport alertmanager-main-0.monitoring\n        addresses: [\"00:00:00:6C:DF:A3 10.16.0.19\"]\nport kube-state-metrics-5d6885d89-4nf8h.monitoring\n        addresses: [\"00:00:00:6F:02:1C 10.16.0.15\"]\nport fake-kubelet-67c55dfd89-pv86k.kube-system\n        addresses: [\"00:00:00:5C:12:E8 10.16.19.177\"]\nport ovn-default-ovn-cluster\n        type: router\n        router-port: ovn-cluster-ovn-default\nrouter 212f73dd-d63d-4d72-864b-a537e9afbee1 (ovn-cluster)\nport ovn-cluster-snat\n        mac: \"00:00:00:7A:82:8F\"\nnetworks: [\"172.22.0.1/16\"]\nport ovn-cluster-join\n        mac: \"00:00:00:F8:18:5A\"\nnetworks: [\"100.64.0.1/16\"]\nport ovn-cluster-central\n        mac: \"00:00:00:4D:8C:F5\"\nnetworks: [\"192.101.0.1/16\"]\nport ovn-cluster-ovn-default\n        mac: \"00:00:00:A3:F8:18\"\nnetworks: [\"10.16.0.1/16\"]\n</code></pre>"},{"location":"en/ops/kubectl-ko/#vsctl-nodename-options","title":"vsctl {nodeName} [options ...]","text":"<p>This command will go to the <code>ovs-ovn</code> container on the corresponding <code>nodeName</code> and execute the corresponding <code>ovs-vsctl</code> command to query and configure <code>vswitchd</code>. For more detailed usage of this command, please refer to the official documentation of the upstream OVS ovs-vsctl(8)\u3002</p> <pre><code># kubectl ko vsctl kube-ovn-01 show\n0d4c4675-c9cc-440a-8c1a-878e17f81b88\n    Bridge br-int\n        fail_mode: secure\n        datapath_type: system\n        Port a2c1a8a8b83a_h\n            Interface a2c1a8a8b83a_h\n        Port \"4fa5c4cbb1a5_h\"\nInterface \"4fa5c4cbb1a5_h\"\nPort ovn-eef07d-0\n            Interface ovn-eef07d-0\n                type: stt\n                options: {csum=\"true\", key=flow, remote_ip=\"192.168.137.178\"}\nPort ovn0\n            Interface ovn0\n                type: internal\n        Port mirror0\n            Interface mirror0\n                type: internal\n        Port ovn-efa253-0\n            Interface ovn-efa253-0\n                type: stt\n                options: {csum=\"true\", key=flow, remote_ip=\"192.168.137.177\"}\nPort br-int\n            Interface br-int\n                type: internal\n    ovs_version: \"2.17.2\"\n</code></pre>"},{"location":"en/ops/kubectl-ko/#ofctl-nodename-options","title":"ofctl {nodeName} [options ...]","text":"<p>This command will go to the <code>ovs-ovn</code> container on the corresponding <code>nodeName</code> and execute the corresponding <code>ovs-ofctl</code> command to query or manage OpenFlow. For more detailed usage of this command, please refer to the official documentation of the upstream OVS ovs-ofctl(8)\u3002</p> <pre><code># kubectl ko ofctl kube-ovn-01 dump-flows br-int\nNXST_FLOW reply (xid=0x4): flags=[more]\ncookie=0xcf3429e6, duration=671791.432s, table=0, n_packets=0, n_bytes=0, idle_age=65534, hard_age=65534, priority=100,in_port=2 actions=load:0x4-&gt;NXM_NX_REG13[],load:0x9-&gt;NXM_NX_REG11[],load:0xb-&gt;NXM_NX_REG12[],load:0x4-&gt;OXM_OF_METADATA[],load:0x1-&gt;NXM_NX_REG14[],resubmit(,8)\ncookie=0xc91413c6, duration=671791.431s, table=0, n_packets=907489, n_bytes=99978275, idle_age=0, hard_age=65534, priority=100,in_port=7 actions=load:0x1-&gt;NXM_NX_REG13[],load:0x9-&gt;NXM_NX_REG11[],load:0xb-&gt;NXM_NX_REG12[],load:0x4-&gt;OXM_OF_METADATA[],load:0x4-&gt;NXM_NX_REG14[],resubmit(,8)\ncookie=0xf180459, duration=671791.431s, table=0, n_packets=17348582, n_bytes=2667811214, idle_age=0, hard_age=65534, priority=100,in_port=6317 actions=load:0xa-&gt;NXM_NX_REG13[],load:0x9-&gt;NXM_NX_REG11[],load:0xb-&gt;NXM_NX_REG12[],load:0x4-&gt;OXM_OF_METADATA[],load:0x9-&gt;NXM_NX_REG14[],resubmit(,8)\ncookie=0x7806dd90, duration=671791.431s, table=0, n_packets=3235428, n_bytes=833821312, idle_age=0, hard_age=65534, priority=100,in_port=1 actions=load:0xd-&gt;NXM_NX_REG13[],load:0x9-&gt;NXM_NX_REG11[],load:0xb-&gt;NXM_NX_REG12[],load:0x4-&gt;OXM_OF_METADATA[],load:0x3-&gt;NXM_NX_REG14[],resubmit(,8)\n...\n</code></pre>"},{"location":"en/ops/kubectl-ko/#dpctl-nodename-options","title":"dpctl {nodeName} [options ...]","text":"<p>This command will go to the <code>ovs-ovn</code> container on the corresponding <code>nodeName</code> and execute the corresponding <code>ovs-dpctl</code> command to query or manage the OVS datapath. For more detailed usage of this command, please refer to the official documentation of the upstream OVS ovs-dpctl(8)\u3002</p> <pre><code># kubectl ko dpctl kube-ovn-01 show\nsystem@ovs-system:\n  lookups: hit:350805055 missed:21983648 lost:73\n  flows: 105\nmasks: hit:1970748791 total:22 hit/pkt:5.29\n  port 0: ovs-system (internal)\nport 1: ovn0 (internal)\nport 2: mirror0 (internal)\nport 3: br-int (internal)\nport 4: stt_sys_7471 (stt: packet_type=ptap)\nport 5: eeb4d9e51b5d_h\n  port 6: a2c1a8a8b83a_h\n  port 7: 4fa5c4cbb1a5_h\n</code></pre>"},{"location":"en/ops/kubectl-ko/#appctl-nodename-options","title":"appctl {nodeName} [options ...]","text":"<p>This command will enter the <code>ovs-ovn</code> container on the corresponding <code>nodeName</code> and execute the corresponding <code>ovs-appctl</code> command to operate the associated daemon process. For more detailed usage of this command, please refer to the official documentation of the upstream OVS ovs-appctl(8)\u3002</p> <pre><code># kubectl ko appctl kube-ovn-01 vlog/list\nconsole    syslog    file\n                 -------    ------    ------\nbacktrace          OFF        ERR       INFO\nbfd                OFF        ERR       INFO\nbond               OFF        ERR       INFO\nbridge             OFF        ERR       INFO\nbundle             OFF        ERR       INFO\nbundles            OFF        ERR       INFO\n...\n</code></pre>"},{"location":"en/ops/kubectl-ko/#tcpdump-namespacepodname-tcpdump-options","title":"tcpdump {namespace/podname} [tcpdump options ...]","text":"<p>This command will enter the <code>kube-ovn-cni</code> container on the machine where <code>namespace/podname</code> is located, and run <code>tcpdump</code> to capture the traffic on the veth NIC of the corresponding container, which can be used to troubleshoot network-related problems.</p> <pre><code># kubectl ko tcpdump default/ds1-l6n7p icmp\n+ kubectl exec -it kube-ovn-cni-wlg4s -n kube-ovn -- tcpdump -nn -i d7176fe7b4e0_h icmp\ntcpdump: verbose output suppressed, use -v or -vv for full protocol decode\nlistening on d7176fe7b4e0_h, link-type EN10MB (Ethernet), capture size 262144 bytes\n06:52:36.619688 IP 100.64.0.3 &gt; 10.16.0.4: ICMP echo request, id 2, seq 1, length 64\n06:52:36.619746 IP 10.16.0.4 &gt; 100.64.0.3: ICMP echo reply, id 2, seq 1, length 64\n06:52:37.619588 IP 100.64.0.3 &gt; 10.16.0.4: ICMP echo request, id 2, seq 2, length 64\n06:52:37.619630 IP 10.16.0.4 &gt; 100.64.0.3: ICMP echo reply, id 2, seq 2, length 64\n06:52:38.619933 IP 100.64.0.3 &gt; 10.16.0.4: ICMP echo request, id 2, seq 3, length 64\n06:52:38.619973 IP 10.16.0.4 &gt; 100.64.0.3: ICMP echo reply, id 2, seq 3, length 64\n</code></pre>"},{"location":"en/ops/kubectl-ko/#trace-arguments","title":"trace [arguments ...]","text":"<p>This command will print the OVN logical flow table and the final Openflow flow table when the Pod/node accesses an address through a specific protocol, so that it make locate flow table related problems during development or troubleshooting much easy.</p> <p>Supported commands:</p> <pre><code>kubectl ko trace {namespace/podname} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp/udp port]\nkubectl ko trace {namespace/podname} {target ip address} [target mac address] arp {request|reply}\nkubectl ko trace {node//nodename} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp/udp port]\nkubectl ko trace {node//nodename} {target ip address} [target mac address] arp {request|reply}\n</code></pre> <p>Example:</p> <pre><code># kubectl ko trace default/ds1-l6n7p 8.8.8.8 icmp\n+ kubectl exec ovn-central-5bc494cb5-np9hm -n kube-ovn -- ovn-trace --ct=new ovn-default 'inport == \"ds1-l6n7p.default\" &amp;&amp; ip.ttl == 64 &amp;&amp; icmp &amp;&amp; eth.src == 0a:00:00:10:00:05 &amp;&amp; ip4.src == 10.16.0.4 &amp;&amp; eth.dst == 00:00:00:B8:CA:43 &amp;&amp; ip4.dst == 8.8.8.8'\n# icmp,reg14=0xf,vlan_tci=0x0000,dl_src=0a:00:00:10:00:05,dl_dst=00:00:00:b8:ca:43,nw_src=10.16.0.4,nw_dst=8.8.8.8,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=0,icmp_code=0\n\ningress(dp=\"ovn-default\", inport=\"ds1-l6n7p.default\")\n-----------------------------------------------------\n 0. ls_in_port_sec_l2 (ovn-northd.c:4143): inport == \"ds1-l6n7p.default\" &amp;&amp; eth.src == {0a:00:00:10:00:05}, priority 50, uuid 39453393\nnext;\n1. ls_in_port_sec_ip (ovn-northd.c:2898): inport == \"ds1-l6n7p.default\" &amp;&amp; eth.src == 0a:00:00:10:00:05 &amp;&amp; ip4.src == {10.16.0.4}, priority 90, uuid 81bcd485\n    next;\n3. ls_in_pre_acl (ovn-northd.c:3269): ip, priority 100, uuid 7b4f4971\n    reg0[0] = 1;\nnext;\n5. ls_in_pre_stateful (ovn-northd.c:3396): reg0[0] == 1, priority 100, uuid 36cdd577\n    ct_next;\n\nct_next(ct_state=new|trk)\n-------------------------\n 6. ls_in_acl (ovn-northd.c:3759): ip &amp;&amp; (!ct.est || (ct.est &amp;&amp; ct_label.blocked == 1)), priority 1, uuid 7608af5b\n    reg0[1] = 1;\nnext;\n10. ls_in_stateful (ovn-northd.c:3995): reg0[1] == 1, priority 100, uuid 2aba1b90\n    ct_commit(ct_label=0/0x1);\nnext;\n16. ls_in_l2_lkup (ovn-northd.c:4470): eth.dst == 00:00:00:b8:ca:43, priority 50, uuid 5c9c3c9f\n    outport = \"ovn-default-ovn-cluster\";\noutput;\n\n...\n</code></pre> <p>If the trace object is a virtual machine running  in Underlay network, additional parameters is needed to specify the destination Mac address.</p> <pre><code>kubectl ko trace default/virt-handler-7lvml 8.8.8.8 82:7c:9f:83:8c:01 icmp\n</code></pre>"},{"location":"en/ops/kubectl-ko/#diagnose-allnodesubnetipports-nodenamesubnetnameproto1-ip1-port1proto2-ip2-port2","title":"diagnose {all|node|subnet|IPPorts} [nodename|subnetName|{proto1}-{IP1}-{Port1},{proto2}-{IP2}-{Port2}]","text":"<p>Diagnose the status of cluster network components and go to the corresponding node's <code>kube-ovn-pinger</code> to detect connectivity and network latency from the current node to other nodes and critical services.</p> <pre><code># kubectl ko diagnose all\nswitch c7cd17e8-ceee-4a91-9bb3-e5a313fe1ece (snat)\nport snat-ovn-cluster\n        type: router\n        router-port: ovn-cluster-snat\nswitch 20e0c6d0-023a-4756-aec5-200e0c60f95d (join)\nport node-liumengxin-ovn3-192.168.137.178\n        addresses: [\"00:00:00:64:FF:A8 100.64.0.4\"]\nport node-liumengxin-ovn1-192.168.137.176\n        addresses: [\"00:00:00:AF:98:62 100.64.0.2\"]\nport join-ovn-cluster\n        type: router\n        router-port: ovn-cluster-join\nswitch 0191705c-f827-427b-9de3-3c3b7d971ba5 (central)\nport central-ovn-cluster\n        type: router\n        router-port: ovn-cluster-central\nswitch 2a45ff05-388d-4f85-9daf-e6fccd5833dc (ovn-default)\nport ovn-default-ovn-cluster\n        type: router\n        router-port: ovn-cluster-ovn-default\n    port prometheus-k8s-1.monitoring\n        addresses: [\"00:00:00:AA:37:DF 10.16.0.23\"]\nrouter 212f73dd-d63d-4d72-864b-a537e9afbee1 (ovn-cluster)\nport ovn-cluster-snat\n        mac: \"00:00:00:7A:82:8F\"\nnetworks: [\"172.22.0.1/16\"]\nport ovn-cluster-join\n        mac: \"00:00:00:F8:18:5A\"\nnetworks: [\"100.64.0.1/16\"]\nport ovn-cluster-central\n        mac: \"00:00:00:4D:8C:F5\"\nnetworks: [\"192.101.0.1/16\"]\nport ovn-cluster-ovn-default\n        mac: \"00:00:00:A3:F8:18\"\nnetworks: [\"10.16.0.1/16\"]\nRouting Policies\n     31000                            ip4.dst == 10.16.0.0/16           allow\n     31000                           ip4.dst == 100.64.0.0/16           allow\n     30000                         ip4.dst == 192.168.137.177         reroute                100.64.0.3\n     30000                         ip4.dst == 192.168.137.178         reroute                100.64.0.4\n     29000                 ip4.src == $ovn.default.fake.6_ip4         reroute               100.64.0.22\n     29000                 ip4.src == $ovn.default.fake.7_ip4         reroute               100.64.0.21\n     29000                 ip4.src == $ovn.default.fake.8_ip4         reroute               100.64.0.23\n     29000 ip4.src == $ovn.default.liumengxin.ovn3.192.168.137.178_ip4         reroute                100.64.0.4\n     20000 ip4.src == $ovn.default.liumengxin.ovn1.192.168.137.176_ip4 &amp;&amp; ip4.dst != $ovn.cluster.overlay.subnets.IPv4         reroute                100.64.0.2\n     20000 ip4.src == $ovn.default.liumengxin.ovn2.192.168.137.177_ip4 &amp;&amp; ip4.dst != $ovn.cluster.overlay.subnets.IPv4         reroute                100.64.0.3\n     20000 ip4.src == $ovn.default.liumengxin.ovn3.192.168.137.178_ip4 &amp;&amp; ip4.dst != $ovn.cluster.overlay.subnets.IPv4         reroute                100.64.0.4\nIPv4 Routes\nRoute Table &lt;main&gt;:\n                0.0.0.0/0                100.64.0.1 dst-ip\nUUID                                    LB                  PROTO      VIP                     IPs\ne9bcfd9d-793e-4431-9073-6dec96b75d71    cluster-tcp-load    tcp        10.100.209.132:10660    192.168.137.176:10660\n                                                            tcp        10.101.239.192:6641     192.168.137.177:6641\n                                                            tcp        10.101.240.101:3000     10.16.0.7:3000\n                                                            tcp        10.103.184.186:6642     192.168.137.177:6642\n35d2b7a5-e3a7-485a-a4b7-b4970eb0e63b    cluster-tcp-sess    tcp        10.100.158.128:8080     10.16.0.10:8080,10.16.0.5:8080,10.16.63.30:8080\n                                                            tcp        10.107.26.215:8080      10.16.0.19:8080,10.16.0.20:8080,10.16.0.21:8080\n                                                            tcp        10.107.26.215:9093      10.16.0.19:9093,10.16.0.20:9093,10.16.0.21:9093\n                                                            tcp        10.98.187.99:8080       10.16.0.22:8080,10.16.0.23:8080\n                                                            tcp        10.98.187.99:9090       10.16.0.22:9090,10.16.0.23:9090\nf43303e4-89aa-4d3e-a3dc-278a552fe27b    cluster-udp-load    udp        10.96.0.10:53           10.16.0.4:53,10.16.0.9:53\n_uuid               : 06776304-5a96-43ed-90c4-c4854c251699\naddresses           : []\nexternal_ids        : {vendor=kube-ovn}\nname                : node_liumengxin_ovn2_192.168.137.177_underlay_v6\n\n_uuid               : 62690625-87d5-491c-8675-9fd83b1f433c\naddresses           : []\nexternal_ids        : {vendor=kube-ovn}\nname                : node_liumengxin_ovn1_192.168.137.176_underlay_v6\n\n_uuid               : b03a9bae-94d5-4562-b34c-b5f6198e180b\naddresses           : [\"10.16.0.0/16\", \"100.64.0.0/16\", \"172.22.0.0/16\", \"192.101.0.0/16\"]\nexternal_ids        : {vendor=kube-ovn}\nname                : ovn.cluster.overlay.subnets.IPv4\n\n_uuid               : e1056f3a-24cc-4666-8a91-75ee6c3c2426\naddresses           : []\nexternal_ids        : {vendor=kube-ovn}\nname                : ovn.cluster.overlay.subnets.IPv6\n\n_uuid               : 3e5d5fff-e670-47b2-a2f5-a39f4698a8c5\naddresses           : []\nexternal_ids        : {vendor=kube-ovn}\nname                : node_liumengxin_ovn3_192.168.137.178_underlay_v6\n_uuid               : 2d85dbdc-d0db-4abe-b19e-cc806d32b492\naction              : drop\ndirection           : from-lport\nexternal_ids        : {}\nlabel               : 0\nlog                 : false\nmatch               : \"inport==@ovn.sg.kubeovn_deny_all &amp;&amp; ip\"\nmeter               : []\nname                : []\noptions             : {}\npriority            : 2003\nseverity            : []\n\n_uuid               : de790cc8-f155-405f-bb32-5a51f30c545f\naction              : drop\ndirection           : to-lport\nexternal_ids        : {}\nlabel               : 0\nlog                 : false\nmatch               : \"outport==@ovn.sg.kubeovn_deny_all &amp;&amp; ip\"\nmeter               : []\nname                : []\noptions             : {}\npriority            : 2003\nseverity            : []\nChassis \"e15ed4d4-1780-4d50-b09e-ea8372ed48b8\"\nhostname: liumengxin-ovn1-192.168.137.176\n    Encap stt\n        ip: \"192.168.137.176\"\noptions: {csum=\"true\"}\nPort_Binding node-liumengxin-ovn1-192.168.137.176\n    Port_Binding perf-6vxkn.default\n    Port_Binding kube-state-metrics-5d6885d89-4nf8h.monitoring\n    Port_Binding alertmanager-main-0.monitoring\n    Port_Binding kube-ovn-pinger-6ftdf.kube-system\n    Port_Binding fake-kubelet-67c55dfd89-pv86k.kube-system\n    Port_Binding prometheus-k8s-0.monitoring\nChassis \"eef07da1-f8ad-4775-b14d-bd6a3b4eb0d5\"\nhostname: liumengxin-ovn3-192.168.137.178\n    Encap stt\n        ip: \"192.168.137.178\"\noptions: {csum=\"true\"}\nPort_Binding kube-ovn-pinger-7twb4.kube-system\n    Port_Binding prometheus-adapter-86df476d87-rl88g.monitoring\n    Port_Binding prometheus-k8s-1.monitoring\n    Port_Binding node-liumengxin-ovn3-192.168.137.178\n    Port_Binding perf-ff475.default\n    Port_Binding alertmanager-main-1.monitoring\n    Port_Binding blackbox-exporter-676d976865-tvsjd.monitoring\nChassis \"efa253c9-494d-4719-83ae-b48ab0f11c03\"\nhostname: liumengxin-ovn2-192.168.137.177\n    Encap stt\n        ip: \"192.168.137.177\"\noptions: {csum=\"true\"}\nPort_Binding grafana-6c4c6b8fb7-pzd2c.monitoring\n    Port_Binding node-liumengxin-ovn2-192.168.137.177\n    Port_Binding alertmanager-main-2.monitoring\n    Port_Binding coredns-6789c94dd8-9jqsz.kube-system\n    Port_Binding coredns-6789c94dd8-25d4r.kube-system\n    Port_Binding prometheus-operator-7bbc99fc8b-wgjm4.monitoring\n    Port_Binding prometheus-adapter-86df476d87-gdxmc.monitoring\n    Port_Binding perf-fjnws.default\n    Port_Binding kube-ovn-pinger-vh2xg.kube-system\nds kube-proxy ready\nkube-proxy ready\ndeployment ovn-central ready\ndeployment kube-ovn-controller ready\nds kube-ovn-cni ready\nds ovs-ovn ready\ndeployment coredns ready\novn-nb leader check ok\novn-sb leader check ok\novn-northd leader check ok\n### kube-ovn-controller recent log\n\n### start to diagnose node liumengxin-ovn1-192.168.137.176\n#### ovn-controller log:\n2022-06-03T00:56:44.897Z|16722|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T01:06:44.912Z|16723|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T01:16:44.925Z|16724|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T01:26:44.936Z|16725|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T01:36:44.959Z|16726|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T01:46:44.974Z|16727|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T01:56:44.988Z|16728|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T02:06:45.001Z|16729|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T02:16:45.025Z|16730|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T02:26:45.040Z|16731|inc_proc_eng|INFO|User triggered force recompute.\n\n#### ovs-vswitchd log:\n2022-06-02T23:03:00.137Z|00079|dpif(handler1)|WARN|system@ovs-system: execute ct(commit,zone=14,label=0/0x1,nat(src)),8 failed (Invalid argument) on packet icmp,vlan_tci=0x0000,dl_src=00:00:00:f8:07:c8,dl_dst=00:00:00:fa:1e:50,nw_src=10.16.0.5,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0 icmp_csum:f9d1\n with metadata skb_priority(0),tunnel(tun_id=0x160017000004,src=192.168.137.177,dst=192.168.137.176,ttl=64,tp_src=38881,tp_dst=7471,flags(csum|key)),skb_mark(0),ct_state(0x21),ct_zone(0xe),ct_tuple4(src=10.16.0.5,dst=10.16.0.10,proto=1,tp_src=8,tp_dst=0),in_port(4) mtu 0\n2022-06-02T23:23:31.840Z|00080|dpif(handler1)|WARN|system@ovs-system: execute ct(commit,zone=14,label=0/0x1,nat(src)),8 failed (Invalid argument) on packet icmp,vlan_tci=0x0000,dl_src=00:00:00:f8:07:c8,dl_dst=00:00:00:fa:1e:50,nw_src=10.16.0.5,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0 icmp_csum:15b2\n with metadata skb_priority(0),tunnel(tun_id=0x160017000004,src=192.168.137.177,dst=192.168.137.176,ttl=64,tp_src=38881,tp_dst=7471,flags(csum|key)),skb_mark(0),ct_state(0x21),ct_zone(0xe),ct_tuple4(src=10.16.0.5,dst=10.16.0.10,proto=1,tp_src=8,tp_dst=0),in_port(4) mtu 0\n2022-06-03T00:09:15.659Z|00081|dpif(handler1)|WARN|system@ovs-system: execute ct(commit,zone=14,label=0/0x1,nat(src)),8 failed (Invalid argument) on packet icmp,vlan_tci=0x0000,dl_src=00:00:00:dc:e3:63,dl_dst=00:00:00:fa:1e:50,nw_src=10.16.63.30,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0 icmp_csum:e5a5\n with metadata skb_priority(0),tunnel(tun_id=0x150017000004,src=192.168.137.178,dst=192.168.137.176,ttl=64,tp_src=9239,tp_dst=7471,flags(csum|key)),skb_mark(0),ct_state(0x21),ct_zone(0xe),ct_tuple4(src=10.16.63.30,dst=10.16.0.10,proto=1,tp_src=8,tp_dst=0),in_port(4) mtu 0\n2022-06-03T00:30:13.409Z|00064|dpif(handler2)|WARN|system@ovs-system: execute ct(commit,zone=14,label=0/0x1,nat(src)),8 failed (Invalid argument) on packet icmp,vlan_tci=0x0000,dl_src=00:00:00:f8:07:c8,dl_dst=00:00:00:fa:1e:50,nw_src=10.16.0.5,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0 icmp_csum:6b4a\n with metadata skb_priority(0),tunnel(tun_id=0x160017000004,src=192.168.137.177,dst=192.168.137.176,ttl=64,tp_src=38881,tp_dst=7471,flags(csum|key)),skb_mark(0),ct_state(0x21),ct_zone(0xe),ct_tuple4(src=10.16.0.5,dst=10.16.0.10,proto=1,tp_src=8,tp_dst=0),in_port(4) mtu 0\n2022-06-03T02:02:33.832Z|00082|dpif(handler1)|WARN|system@ovs-system: execute ct(commit,zone=14,label=0/0x1,nat(src)),8 failed (Invalid argument) on packet icmp,vlan_tci=0x0000,dl_src=00:00:00:f8:07:c8,dl_dst=00:00:00:fa:1e:50,nw_src=10.16.0.5,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0 icmp_csum:a819\n with metadata skb_priority(0),tunnel(tun_id=0x160017000004,src=192.168.137.177,dst=192.168.137.176,ttl=64,tp_src=38881,tp_dst=7471,flags(csum|key)),skb_mark(0),ct_state(0x21),ct_zone(0xe),ct_tuple4(src=10.16.0.5,dst=10.16.0.10,proto=1,tp_src=8,tp_dst=0),in_port(4) mtu 0\n\n#### ovs-vsctl show results:\n0d4c4675-c9cc-440a-8c1a-878e17f81b88\n    Bridge br-int\n        fail_mode: secure\n        datapath_type: system\n        Port a2c1a8a8b83a_h\n            Interface a2c1a8a8b83a_h\n        Port \"4fa5c4cbb1a5_h\"\nInterface \"4fa5c4cbb1a5_h\"\nPort ovn-eef07d-0\n            Interface ovn-eef07d-0\n                type: stt\n                options: {csum=\"true\", key=flow, remote_ip=\"192.168.137.178\"}\nPort ovn0\n            Interface ovn0\n                type: internal\n        Port \"04d03360e9a0_h\"\nInterface \"04d03360e9a0_h\"\nPort eeb4d9e51b5d_h\n            Interface eeb4d9e51b5d_h\n        Port mirror0\n            Interface mirror0\n                type: internal\n        Port \"8e5d887ccd80_h\"\nInterface \"8e5d887ccd80_h\"\nPort ovn-efa253-0\n            Interface ovn-efa253-0\n                type: stt\n                options: {csum=\"true\", key=flow, remote_ip=\"192.168.137.177\"}\nPort \"17512d5be1f1_h\"\nInterface \"17512d5be1f1_h\"\nPort br-int\n            Interface br-int\n                type: internal\n    ovs_version: \"2.17.2\"\n\n#### pinger diagnose results:\nI0603 10:35:04.349404   17619 pinger.go:19]\n-------------------------------------------------------------------------------\nKube-OVN:\n  Version:       v1.13.0\n  Build:         2022-04-24_08:02:50\n  Commit:        git-73f9d15\n  Go Version:    go1.17.8\n  Arch:          amd64\n-------------------------------------------------------------------------------\nI0603 10:35:04.376797   17619 config.go:166] pinger config is &amp;{KubeConfigFile: KubeClient:0xc000493380 Port:8080 DaemonSetNamespace:kube-system DaemonSetName:kube-ovn-pinger Interval:5 Mode:job ExitCode:0 InternalDNS:kubernetes.default ExternalDNS: NodeName:liumengxin-ovn1-192.168.137.176 HostIP:192.168.137.176 PodName:kube-ovn-pinger-6ftdf PodIP:10.16.0.10 PodProtocols:[IPv4] ExternalAddress: NetworkMode:kube-ovn PollTimeout:2 PollInterval:15 SystemRunDir:/var/run/openvswitch DatabaseVswitchName:Open_vSwitch DatabaseVswitchSocketRemote:unix:/var/run/openvswitch/db.sock DatabaseVswitchFileDataPath:/etc/openvswitch/conf.db DatabaseVswitchFileLogPath:/var/log/openvswitch/ovsdb-server.log DatabaseVswitchFilePidPath:/var/run/openvswitch/ovsdb-server.pid DatabaseVswitchFileSystemIDPath:/etc/openvswitch/system-id.conf ServiceVswitchdFileLogPath:/var/log/openvswitch/ovs-vswitchd.log ServiceVswitchdFilePidPath:/var/run/openvswitch/ovs-vswitchd.pid ServiceOvnControllerFileLogPath:/var/log/ovn/ovn-controller.log ServiceOvnControllerFilePidPath:/var/run/ovn/ovn-controller.pid}\nI0603 10:35:04.449166   17619 exporter.go:75] liumengxin-ovn1-192.168.137.176: exporter connect successfully\nI0603 10:35:04.554011   17619 ovn.go:21] ovs-vswitchd and ovsdb are up\nI0603 10:35:04.651293   17619 ovn.go:33] ovn_controller is up\nI0603 10:35:04.651342   17619 ovn.go:39] start to check port binding\nI0603 10:35:04.749613   17619 ovn.go:135] chassis id is 1d7f3d6c-eec5-4b3c-adca-2969d9cdfd80\nI0603 10:35:04.763487   17619 ovn.go:49] port in sb is [node-liumengxin-ovn1-192.168.137.176 perf-6vxkn.default kube-state-metrics-5d6885d89-4nf8h.monitoring alertmanager-main-0.monitoring kube-ovn-pinger-6ftdf.kube-system fake-kubelet-67c55dfd89-pv86k.kube-system prometheus-k8s-0.monitoring]\nI0603 10:35:04.763583   17619 ovn.go:61] ovs and ovn-sb binding check passed\nI0603 10:35:05.049309   17619 ping.go:259] start to check apiserver connectivity\nI0603 10:35:05.053666   17619 ping.go:268] connect to apiserver success in 4.27ms\nI0603 10:35:05.053786   17619 ping.go:129] start to check pod connectivity\nI0603 10:35:05.249590   17619 ping.go:159] ping pod: kube-ovn-pinger-6ftdf 10.16.0.10, count: 3, loss count 0, average rtt 16.30ms\nI0603 10:35:05.354135   17619 ping.go:159] ping pod: kube-ovn-pinger-7twb4 10.16.63.30, count: 3, loss count 0, average rtt 1.81ms\nI0603 10:35:05.458460   17619 ping.go:159] ping pod: kube-ovn-pinger-vh2xg 10.16.0.5, count: 3, loss count 0, average rtt 1.92ms\nI0603 10:35:05.458523   17619 ping.go:83] start to check node connectivity\n</code></pre> <p>If the target of diagnose is specified as subnet, the script will create a daemonset on the subnet, and <code>kube-ovn-pinger</code> will detect the connectivity and network delay of all pods in this daemonset, and automatically destroy the daemonset after the test.</p> <p>If the target of diagnose is specified as IPPorts, the script will let each <code>kube-ovn-pinger</code> pod detect whether the target protocol, IP, and Port are reachable.</p>"},{"location":"en/ops/kubectl-ko/#tuning-install-fastpathlocal-install-fastpathremove-fastpathinstall-sttlocal-install-sttremove-stt-centos7centos8-kernel-devel-version","title":"tuning {install-fastpath|local-install-fastpath|remove-fastpath|install-stt|local-install-stt|remove-stt} {centos7|centos8}} [kernel-devel-version]","text":"<p>This command performs performance tuning related operations, please refer to Performance Tunning.</p>"},{"location":"en/ops/kubectl-ko/#reload","title":"reload","text":"<p>This command restarts all Kube-OVN related components:</p> <pre><code># kubectl ko reload\npod \"ovn-central-8684dd94bd-vzgcr\" deleted\nWaiting for deployment \"ovn-central\" rollout to finish: 0 of 1 updated replicas are available...\ndeployment \"ovn-central\" successfully rolled out\npod \"ovs-ovn-bsnvz\" deleted\npod \"ovs-ovn-m9b98\" deleted\npod \"kube-ovn-controller-8459db5ff4-64c62\" deleted\nWaiting for deployment \"kube-ovn-controller\" rollout to finish: 0 of 1 updated replicas are available...\ndeployment \"kube-ovn-controller\" successfully rolled out\npod \"kube-ovn-cni-2klnh\" deleted\npod \"kube-ovn-cni-t2jz4\" deleted\nWaiting for daemon set \"kube-ovn-cni\" rollout to finish: 0 of 2 updated pods are available...\nWaiting for daemon set \"kube-ovn-cni\" rollout to finish: 1 of 2 updated pods are available...\ndaemon set \"kube-ovn-cni\" successfully rolled out\npod \"kube-ovn-pinger-ln72z\" deleted\npod \"kube-ovn-pinger-w8lrk\" deleted\nWaiting for daemon set \"kube-ovn-pinger\" rollout to finish: 0 of 2 updated pods are available...\nWaiting for daemon set \"kube-ovn-pinger\" rollout to finish: 1 of 2 updated pods are available...\ndaemon set \"kube-ovn-pinger\" successfully rolled out\npod \"kube-ovn-monitor-7fb67d5488-7q6zb\" deleted\nWaiting for deployment \"kube-ovn-monitor\" rollout to finish: 0 of 1 updated replicas are available...\ndeployment \"kube-ovn-monitor\" successfully rolled out\n</code></pre>"},{"location":"en/ops/kubectl-ko/#log","title":"log","text":"<p>Using this command will capture the logs of Kube-OVN, OVN, Openvswitch on all nodes of kube-ovn and some debug information commonly used in linux.</p> <pre><code># kubectl ko log all\nCollecting kube-ovn logging files\nCollecting ovn logging files\nCollecting openvswitch logging files\nCollecting linux dmesg files\nCollecting linux iptables-legacy files\nCollecting linux iptables-nft files\nCollecting linux route files\nCollecting linux link files\nCollecting linux neigh files\nCollecting linux memory files\nCollecting linux top files\nCollecting linux sysctl files\nCollecting linux netstat files\nCollecting linux addr files\nCollecting linux ipset files\nCollecting linux tcp files\nCollected files have been saved in the directory /root/kubectl-ko-log\n</code></pre> <p>The directory is as follows:</p> <pre><code># tree kubectl-ko-log/\nkubectl-ko-log/\n|-- kube-ovn-control-plane\n|   |-- kube-ovn\n|   |   |-- kube-ovn-cni.log\n|   |   |-- kube-ovn-monitor.log\n|   |   `-- kube-ovn-pinger.log\n|   |-- linux\n|   |   |-- addr.log\n|   |   |-- dmesg.log\n|   |   |-- ipset.log\n|   |   |-- iptables-legacy.log\n|   |   |-- iptables-nft.log\n|   |   |-- link.log\n|   |   |-- memory.log\n|   |   |-- neigh.log\n|   |   |-- netstat.log\n|   |   |-- route.log\n|   |   |-- sysctl.log\n|   |   |-- tcp.log\n|   |   `-- top.log\n|   |-- openvswitch\n|   |   |-- ovs-vswitchd.log\n|   |   `-- ovsdb-server.log\n|   `-- ovn\n|       |-- ovn-controller.log\n|       |-- ovn-northd.log\n|       |-- ovsdb-server-nb.log\n|       `-- ovsdb-server-sb.log\n</code></pre>"},{"location":"en/ops/kubectl-ko/#perf-image","title":"perf [image]","text":"<p>This command will test some performance indicators of Kube-OVN as follows:</p> <ol> <li>The performance indicators of the container network;</li> <li>Hostnetwork network performance indicators;</li> <li>Container network multicast packet performance indicators;</li> <li>Time required for OVN-NB, OVN-SB, and OVN-Northd leader deletion recovery. The parameter image is used to specify the image used by the performance test pod. By default, it is <code>kubeovn/test:v1.12.0</code>. This parameter is mainly set for offline scenarios, and the image name may change when the image is pulled to the intranet environment.</li> </ol> <pre><code># kubectl ko perf\n============================== Prepareing Performance Test Resources ===============================\npod/test-client created\npod/test-host-client created\npod/test-server created\npod/test-host-server created\nservice/test-server created\npod/test-client condition met\npod/test-host-client condition met\npod/test-host-server condition met\npod/test-server condition met\n====================================================================================================\n============================ Start Pod Network Unicast Performance Test ============================\nSize            TCP Latency     TCP Bandwidth   UDP Latency     UDP Lost Rate   UDP Bandwidth\n64              82.8 us         97.7 Mbits/sec  67.6 us         (0%)            8.42 Mbits/sec\n128             85.4 us         167 Mbits/sec   67.2 us         (0%)            17.2 Mbits/sec\n512             85.8 us         440 Mbits/sec   68.7 us         (0%)            68.4 Mbits/sec\n1k              85.1 us         567 Mbits/sec   68.7 us         (0%)            134 Mbits/sec\n4k              138 us          826 Mbits/sec   78.1 us         (1.4%)          503 Mbits/sec\n====================================================================================================\n=============================== Start Host Network Performance Test ================================\nSize            TCP Latency     TCP Bandwidth   UDP Latency     UDP Lost Rate   UDP Bandwidth\n64              49.7 us         120 Mbits/sec   37.9 us         (0%)            18.6 Mbits/sec\n128             49.7 us         200 Mbits/sec   38.1 us         (0%)            35.5 Mbits/sec\n512             51.9 us         588 Mbits/sec   38.9 us         (0%)            142 Mbits/sec\n1k              51.7 us         944 Mbits/sec   37.2 us         (0%)            279 Mbits/sec\n4k              74.9 us         1.66 Gbits/sec  39.9 us         (0%)            1.20 Gbits/sec\n====================================================================================================\n============================== Start Service Network Performance Test ==============================\nSize            TCP Latency     TCP Bandwidth   UDP Latency     UDP Lost Rate   UDP Bandwidth\n64              111 us          96.3 Mbits/sec  88.4 us         (0%)            7.59 Mbits/sec\n128             83.7 us         150 Mbits/sec   69.2 us         (0%)            16.9 Mbits/sec\n512             87.4 us         374 Mbits/sec   75.8 us         (0%)            60.9 Mbits/sec\n1k              88.2 us         521 Mbits/sec   73.1 us         (0%)            123 Mbits/sec\n4k              148 us          813 Mbits/sec   77.6 us         (0.0044%)       451 Mbits/sec\n====================================================================================================\n=========================== Start Pod Multicast Network Performance Test ===========================\nSize            UDP Latency     UDP Lost Rate   UDP Bandwidth\n64              0.014 ms        (0.17%)         5.80 Mbits/sec\n128             0.012 ms        (0%)            11.4 Mbits/sec\n512             0.016 ms        (0%)            46.1 Mbits/sec\n1k              0.023 ms        (0.073%)        89.8 Mbits/sec\n4k              0.035 ms        (1.3%)          126 Mbits/sec\n====================================================================================================\n============================= Start Host Multicast Network Performance =============================\nSize            UDP Latency     UDP Lost Rate   UDP Bandwidth\n64              0.007 ms        (0%)            9.95 Mbits/sec\n128             0.005 ms        (0%)            21.8 Mbits/sec\n512             0.008 ms        (0%)            86.8 Mbits/sec\n1k              0.013 ms        (0.045%)        168 Mbits/sec\n4k              0.010 ms        (0.31%)         242 Mbits/sec\n====================================================================================================\n================================== Start Leader Recover Time Test ==================================\nDelete ovn central nb pod\npod \"ovn-central-5cb9c67d75-tlz9w\" deleted\nWaiting for ovn central nb pod running\n=============================== OVN nb Recovery takes 3.305236803 s ================================\nDelete ovn central sb pod\npod \"ovn-central-5cb9c67d75-szx4c\" deleted\nWaiting for ovn central sb pod running\n=============================== OVN sb Recovery takes 3.462698535 s ================================\nDelete ovn central northd pod\npod \"ovn-central-5cb9c67d75-zqmqv\" deleted\nWaiting for ovn central northd pod running\n============================= OVN northd Recovery takes 2.691291403 s ==============================\n====================================================================================================\n================================= Remove Performance Test Resource =================================\nrm -f unicast-test-client.log\nrm -f unicast-test-host-client.log\nrm -f unicast-test-client.log\nkubectl ko nbctl lb-del test-server\nrm -f multicast-test-server.log\nkubectl exec ovs-ovn-gxdrf -n kube-system -- ip maddr del 01:00:5e:00:00:64 dev eth0\nkubectl exec ovs-ovn-h57bf -n kube-system -- ip maddr del 01:00:5e:00:00:64 dev eth0\nrm -f multicast-test-host-server.log\npod \"test-client\" deleted\npod \"test-host-client\" deleted\npod \"test-host-server\" deleted\npod \"test-server\" deleted\nservice \"test-server\" deleted\n====================================================================================================\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/ops/recover-db/","title":"OVN DB Backup and Recovery","text":"<p>This document describes how to perform database backups and how to perform cluster recovery from existing database files in different situations.</p>"},{"location":"en/ops/recover-db/#database-backup","title":"Database Backup","text":"<p>The database files can be backed up for recovery in case of failure. Use the backup command of the kubectl plugin:</p> <pre><code># kubectl ko nb backup\ntar: Removing leading `/' from member names\nbackup ovn-nb db to /root/ovnnb_db.060223191654183154.backup\n\n# kubectl ko sb backup\ntar: Removing leading `/' from member names\nbackup ovn-nb db to /root/ovnsb_db.060223191654183154.backup\n</code></pre>"},{"location":"en/ops/recover-db/#cluster-partial-nodes-failure-recovery","title":"Cluster Partial Nodes Failure Recovery","text":"<p>If some nodes in the cluster are working abnormally due to power failure, file system failure or lack of disk space, but the cluster is still working normally, you can recover it by following the steps below.</p>"},{"location":"en/ops/recover-db/#check-the-logs-to-confirm-status","title":"Check the Logs to Confirm Status","text":"<p>Check the log in <code>/var/log/ovn/ovn-northd.log</code>, if it shows similar error as follows, you can make sue that there is an exception in the database:</p> <pre><code> * ovn-northd is not running\novsdb-server: ovsdb error: error reading record 2739 from OVN_Northbound log: record 2739 advances commit index to 6308 but last log index is 6307\n* Starting ovsdb-nb\n</code></pre>"},{"location":"en/ops/recover-db/#kick-node-from-cluster","title":"Kick Node from Cluster","text":"<p>Select the corresponding database for the operation based on whether the log prompt is <code>OVN_Northbound</code> or <code>OVN_Southbound</code>. The above log prompt is <code>OVN_Northbound</code> then for ovn-nb do the following:</p> <pre><code># kubectl ko nb status\n9182\nName: OVN_Northbound\nCluster ID: e75f (e75fa340-49ed-45ab-990e-26cb865ebc85)\nServer ID: 9182 (9182e8dd-b5b0-4dd8-8518-598cc1e374f3)\nAddress: tcp:[10.0.128.61]:6643\nStatus: cluster member\nRole: leader\nTerm: 1454\nLeader: self\nVote: self\n\nLast Election started 1732603 ms ago, reason: timeout\nLast Election won: 1732587 ms ago\nElection timer: 1000\nLog: [7332, 12512]\nEntries not yet committed: 1\nEntries not yet applied: 1\nConnections: -&gt;f080 &lt;-f080 &lt;-e631 -&gt;e631\nDisconnections: 1\nServers:\n    f080 (f080 at tcp:[10.0.129.139]:6643) next_index=12512 match_index=12510 last msg 63 ms ago\n    9182 (9182 at tcp:[10.0.128.61]:6643) (self) next_index=10394 match_index=12510\ne631 (e631 at tcp:[10.0.131.173]:6643) next_index=12512 match_index=0\n</code></pre> <p>Kick abnormal nodes from the cluster:</p> <pre><code>kubectl ko nb kick e631\n</code></pre> <p>Log in to the abnormal node and delete the database file:</p> <pre><code>mv /etc/origin/ovn/ovnnb_db.db /tmp\n</code></pre> <p>Delete the <code>ovn-central</code> pod of the corresponding node and wait for the cluster to recover\uff1a</p> <pre><code>kubectl delete pod -n kube-system ovn-central-xxxx\n</code></pre>"},{"location":"en/ops/recover-db/#recover-when-total-cluster-failed","title":"Recover when Total Cluster Failed","text":"<p>If the majority of the cluster nodes are broken and the leader cannot be elected, please refer to the following steps to recover.</p>"},{"location":"en/ops/recover-db/#stop-ovn-central","title":"Stop ovn-central","text":"<p>Record the current replicas of <code>ovn-central</code> and stop <code>ovn-central</code> to avoid new database changes that affect recovery:</p> <pre><code>kubectl scale deployment -n kube-system ovn-central --replicas=0\n</code></pre>"},{"location":"en/ops/recover-db/#select-a-backup","title":"Select a Backup","text":"<p>As most of the nodes are damaged, the cluster needs to be rebuilt by recovering from one of the database files. If you have previously backed up the database you can use the previous backup file to restore it. If not you can use the following steps to generate a backup from an existing file.</p> <p>Since the database file in the default folder is a cluster format database file containing information about the current cluster, you can't rebuild the database directly with this file, you need to use <code>ovsdb-tool cluster-to-standalone</code> to convert the format.</p> <p>Select the first node in the <code>ovn-central</code> environment variable <code>NODE_IPS</code> to restore the database files. If the database file of the first node is corrupted, copy the file from the other machine <code>/etc/origin/ovn</code> to the first machine. Run the following command to generate a database file backup.</p> <pre><code>docker run -it -v /etc/origin/ovn:/etc/ovn kubeovn/kube-ovn:v1.13.0 bash\ncd /etc/ovn/\novsdb-tool cluster-to-standalone ovnnb_db_standalone.db ovnnb_db.db\novsdb-tool cluster-to-standalone ovnsb_db_standalone.db ovnsb_db.db\n</code></pre>"},{"location":"en/ops/recover-db/#delete-the-database-files-on-all-ovn-central-nodes","title":"Delete the Database Files on All ovn-central Nodes","text":"<p>In order to avoid rebuilding the cluster with the wrong data, the existing database files need to be cleaned up:</p> <pre><code>mv /etc/origin/ovn/ovnnb_db.db /tmp\nmv /etc/origin/ovn/ovnsb_db.db /tmp\n</code></pre>"},{"location":"en/ops/recover-db/#recovering-database-cluster","title":"Recovering Database Cluster","text":"<p>Rename the backup databases to <code>ovnnb_db.db</code> and <code>ovnsb_db.db</code> respectively, and copy them to the <code>/etc/origin/ovn/</code> directory of the first machine in the <code>ovn-central</code> environment variable <code>NODE_IPS</code>\uff1a</p> <pre><code>mv /etc/origin/ovn/ovnnb_db_standalone.db /etc/origin/ovn/ovnnb_db.db\nmv /etc/origin/ovn/ovnsb_db_standalone.db /etc/origin/ovn/ovnsb_db.db\n</code></pre> <p>Restore the number of replicas of <code>ovn-central</code>\uff1a</p> <pre><code>kubectl scale deployment -n kube-system ovn-central --replicas=3\nkubectl rollout status deployment/ovn-central -n kube-system\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/reference/architecture/","title":"Architecture","text":"<p>This document describes the general architecture of Kube-OVN, the functionality of each component and how they interact with each other.</p> <p>Overall, Kube-OVN serves as a bridge between Kubernetes and OVN, combining proven SDN with Cloud Native. This means that Kube-OVN not only implements network specifications under Kubernetes, such as CNI, Service and Networkpolicy, but also brings a large number of SDN domain capabilities to cloud-native, such as logical switches, logical routers, VPCs, gateways, QoS, ACLs and traffic mirroring.</p> <p>Kube-OVN also maintains a good openness to integrate with many technology solutions, such as Cilium, Submariner, Prometheus, KubeVirt, etc.</p>"},{"location":"en/reference/architecture/#component-introduction","title":"Component Introduction","text":"<p>The components of Kube-OVN can be broadly divided into three categories.</p> <ul> <li>Upstream OVN/OVS components.</li> <li>Core Controller and Agent.</li> <li>Monitoring, operation and maintenance tools and extension components.</li> </ul> <p></p>"},{"location":"en/reference/architecture/#upstream-ovnovs-components","title":"Upstream OVN/OVS Components","text":"<p>This type of component comes from the OVN/OVS community with specific modifications for Kube-OVN usage scenarios. OVN/OVS itself is a mature SDN system for managing virtual machines and containers, and we strongly recommend that users interested in the Kube-OVN implementation read ovn-architecture(7) first to understand what OVN is and how to integrate with it. Kube-OVN uses the northbound interface of OVN to create and coordinate virtual networks and map the network concepts into Kubernetes.</p> <p>All OVN/OVS-related components have been packaged into images and are ready to run in Kubernetes.</p>"},{"location":"en/reference/architecture/#ovn-central","title":"ovn-central","text":"<p>The <code>ovn-central</code> Deployment runs the control plane components of OVN, including <code>ovn-nb</code>, <code>ovn-sb</code>, and <code>ovn-northd</code>.</p> <ul> <li><code>ovn-nb</code>: Saves the virtual network configuration and provides an API for virtual network management. <code>kube-ovn-controller</code> will mainly interact with <code>ovn-nb</code> to configure the virtual network.</li> <li><code>ovn-sb</code>: Holds the logical flow table generated from the logical network of <code>ovn-nb</code>, as well as the actual physical network state of each node.</li> <li><code>ovn-northd</code>: translates the virtual network of <code>ovn-nb</code> into a logical flow table in <code>ovn-sb</code>.</li> </ul> <p>Multiple instances of <code>ovn-central</code> will synchronize data via the Raft protocol to ensure high availability.</p>"},{"location":"en/reference/architecture/#ovs-ovn","title":"ovs-ovn","text":"<p><code>ovs-ovn</code> runs as a DaemonSet on each node, with <code>openvswitch</code>, <code>ovsdb</code>, and <code>ovn-controller</code> running inside the Pod. These components act as agents for <code>ovn-central</code> to translate logical flow tables into real network configurations.</p>"},{"location":"en/reference/architecture/#core-controller-and-agent","title":"Core Controller and Agent","text":"<p>This part is the core component of Kube-OVN, serving as a bridge between OVN and Kubernetes, bridging the two systems and translating network concepts between them. Most of the core functions are implemented in these components.</p>"},{"location":"en/reference/architecture/#kube-ovn-controller","title":"kube-ovn-controller","text":"<p>This component performs the translation of all resources within Kubernetes to OVN resources and acts as the control plane for the entire Kube-OVN system. The <code>kube-ovn-controller</code> listens for events on all resources related to network functionality and updates the logical network within the OVN based on resource changes. The main resources listened including:</p> <p>Pod\uff0cService\uff0cEndpoint\uff0cNode\uff0cNetworkPolicy\uff0cVPC\uff0cSubnet\uff0cVlan\uff0cProviderNetwork\u3002</p> <p>Taking the Pod event as an example, <code>kube-ovn-controller</code> listens to the Pod creation event, allocates the address via the built-in in-memory IPAM function, and calls <code>ovn-central</code> to create logical ports, static routes and possible ACL rules. Next, <code>kube-ovn-controller</code> writes the assigned address and subnet information such as CIDR, gateway, route, etc. to the annotation of the Pod. This annotation is then read by <code>kube-ovn-cni</code> and used to configure the local network.</p>"},{"location":"en/reference/architecture/#kube-ovn-cni","title":"kube-ovn-cni","text":"<p>This component runs on each node as a DaemonSet, implements the CNI interface, and operates the local OVS to configure the local network.</p> <p>This DaemonSet copies the <code>kube-ovn</code> binary to each machine as a tool for interaction between <code>kubelet</code> and <code>kube-ovn-cni</code>. This binary sends the corresponding CNI request to <code>kube-ovn-cni</code> for further operation. The binary will be copied to the <code>/opt/cni/bin</code> directory by default.</p> <p><code>kube-ovn-cni</code> will configure the specific network to perform the appropriate traffic operations, and the main tasks including:</p> <ol> <li>Config <code>ovn-controller</code> and <code>vswitchd</code>.</li> <li>Handle CNI Add/Del requests:<ol> <li>Create or delete veth pair and bind or unbind to OVS ports.</li> <li>Configure OVS ports</li> <li>Update host iptables/ipset/route rules.</li> </ol> </li> <li>Dynamically update the network QoS.</li> <li>Create and configure the <code>ovn0</code> NIC to connect the container network and the host network.</li> <li>Configure the host NIC to implement Vlan/Underlay/EIP.</li> <li>Dynamically config inter-cluster gateways.</li> </ol>"},{"location":"en/reference/architecture/#monitoring-operation-and-maintenance-tools-and-extension-components","title":"Monitoring, Operation and Maintenance Tools and Extension Components","text":"<p>These components provide monitoring, diagnostics, operations tools, and external interface to extend the core network capabilities of Kube-OVN and simplify daily operations and maintenance.</p>"},{"location":"en/reference/architecture/#kube-ovn-speaker","title":"kube-ovn-speaker","text":"<p>This component is a DaemonSet running on a specific labeled nodes that publish routes to the external, allowing external access to the container directly through the Pod IP.</p> <p>For more information on how to use it, please refer to BGP Support.</p>"},{"location":"en/reference/architecture/#kube-ovn-pinger","title":"kube-ovn-pinger","text":"<p>This component is a DaemonSet running on each node to collect OVS status information, node network quality, network latency, etc. The monitoring metrics collected can be found in Metrics.</p>"},{"location":"en/reference/architecture/#kube-ovn-monitor","title":"kube-ovn-monitor","text":"<p>This component collects OVN status information and the monitoring metrics, all metrics can be found in Metrics.</p>"},{"location":"en/reference/architecture/#kubectl-ko","title":"kubectl-ko","text":"<p>This component is a kubectl plugin, which can quickly run common operations, for more usage, please refer to [kubectl plugin].(../ops/kubectl-ko.en.md)\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/reference/dev-env/","title":"Development Setup","text":""},{"location":"en/reference/dev-env/#environmental-preparation","title":"Environmental Preparation","text":"<p>Kube-OVN uses Golang 1.20 to develop and Go Modules to manage dependency, please check env <code>GO111MODULE=\"on\"</code>\u3002</p> <p>gosec is used to scan for code security related issues and requires to be installed in the development environment:</p> <pre><code>go get github.com/securego/gosec/v2/cmd/gosec\n</code></pre> <p>To reduce the size of the final generated image, Kube-OVN uses some of the Docker buildx experimental features, please update Docker to the latest version and enable buildx:</p> <pre><code>docker buildx create --use\n</code></pre>"},{"location":"en/reference/dev-env/#build-image","title":"Build Image","text":"<p>Use the following command to download the code and generate the image required to run Kube-OVN:</p> <pre><code>git clone https://github.com/kubeovn/kube-ovn.git\ncd kube-ovn\nmake release\n</code></pre> <p>To build an image to run in an ARM environment, run the following command:</p> <pre><code>make release-arm\n</code></pre>"},{"location":"en/reference/dev-env/#building-the-base-image","title":"Building the Base Image","text":"<p>If you need to change the operating system version, dependencies, OVS/OVN code, etc., you need to rebuild the base image.</p> <p>The Dockerfile used for the base image is <code>dist/images/Dockerfile.base</code>.</p> <p>Build instructions:</p> <pre><code># build x86 base image\nmake base-amd64\n\n# build arm base image\nmake base-arm64\n</code></pre>"},{"location":"en/reference/dev-env/#run-e2e","title":"Run E2E","text":"<p>Kube-OVN uses KIND to build local Kubernetes cluster, j2cli to render templates, and Ginkgo to run test cases. Please refer to the relevant documentation for dependency installation.</p> <p>Run E2E locally:</p> <pre><code>make kind-init\nmake kind-install\nmake e2e\n</code></pre> <p>To run the Underlay E2E test, run the following commands:</p> <pre><code>make kind-init\nmake kind-install-underlay\nmake e2e-underlay-single-nic\n</code></pre> <p>To run the ovn vpc nat gw eip, fip, snat, dnat E2E test, run the following commands:</p> <pre><code>make kind-init\nmake kind-install\nmake ovn-vpc-nat-gw-conformance-e2e\n</code></pre> <p>To run the iptables vpc nat gw eip, fip, snat, dnat E2E test, run the following commands:</p> <pre><code>make kind-init\nmake kind-install\nmake kind-install-vpc-nat-gw\nmake iptables-vpc-nat-gw-conformance-e2e\n</code></pre> <p>To run the loadbalancer service E2E test, run the following commands:</p> <pre><code>make kind-init\nmake kind-install\nmake kind-install-lb-svc\nmake kube-ovn-lb-svc-conformance-e2e\n</code></pre> <p>To clean, run the following commands:</p> <pre><code>make kind-clean\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/reference/document-convention/","title":"Document Specification","text":"<p>In order to ensure a consistent document style, please follow the following style guidelines when submitting documents.</p>"},{"location":"en/reference/document-convention/#punctuation","title":"Punctuation","text":"<p>All punctuation in the text content in Chinese documents should use Chinese format punctuation, and all text content in English documents should use English punctuation.</p> BadGood   Here is a one-click installation script that can help you quickly install a highly available, production-ready container network.    Here is a one-click installation script that can help you quickly install a highly available, production-ready container network.   <p>English numbers and Chinese characters should be separated by spaces.</p> BadGood   Kube-OVN provides a one-click installation script to install version 1.10 of Kube-OVN.    Kube-OVN provides a one-click installation script to install version 1.10 of Kube-OVN.   <p>Example content should start with <code>:</code>, other sentences should end with <code>.</code> End.</p> BadGood   Please confirm that the environment configuration is correct before installation  Download the installation script using the command below.  <pre><code>wget 127.0.0.1\n</code></pre>   Please confirm that the environment configuration is correct before installation.  Download the installation script using the following command:  <pre><code>wget 127.0.0.1\n</code></pre>"},{"location":"en/reference/document-convention/#code-block","title":"Code Block","text":"<p>yaml code blocks need to be identified as yaml.</p> BadGood <pre><code>````\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n     name: attach-subnet\n````\n</code></pre> <pre><code>````yaml\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n     name: attach-subnet\n````\n</code></pre> <p>Command-line manipulation example code blocks need to be identified as bash.</p> BadGood <pre><code>````\nwget 127.0.0.1\n````\n</code></pre> <pre><code>````bash\nwget 127.0.0.1\n````\n</code></pre> <p>If the command line operation example contains output content, the executed command needs to start with <code>#</code> to distinguish input from output.</p> BadGood <pre><code>oilbeater@macdeMac-3 ~ ping 114.114.114.114 -c 3\nPING 114.114.114.114 (114.114.114.114): 56 data bytes\n64 bytes from 114.114.114.114: icmp_seq=0 ttl=83 time=10.429 ms\n64 bytes from 114.114.114.114: icmp_seq=1 ttl=79 time=11.360 ms\n64 bytes from 114.114.114.114: icmp_seq=2 ttl=76 time=10.794 ms\n\n--- 114.114.114.114 ping statistics ---\n3 packets transmitted, 3 packets received, 0.0% packet loss\nround-trip min/avg/max/stddev = 10.429/10.861/11.360/0.383 ms\n</code></pre> <pre><code># ping 114.114.114.114 -c 3\nPING 114.114.114.114 (114.114.114.114): 56 data bytes\n64 bytes from 114.114.114.114: icmp_seq=0 ttl=83 time=10.429 ms\n64 bytes from 114.114.114.114: icmp_seq=1 ttl=79 time=11.360 ms\n64 bytes from 114.114.114.114: icmp_seq=2 ttl=76 time=10.794 ms\n\n--- 114.114.114.114 ping statistics ---\n3 packets transmitted, 3 packets received, 0.0% packet loss\nround-trip min/avg/max/stddev = 10.429/10.861/11.360/0.383 ms\n</code></pre> <p>If the command line operation example only contains execution commands and no output results, multiple commands do not need to start with <code>#</code>.</p> BadGood <pre><code># mv /etc/origin/ovn/ovnnb_db.db /tmp\n# mv /etc/origin/ovn/ovnsb_db.db /tmp\n</code></pre> <pre><code>mv /etc/origin/ovn/ovnnb_db.db /tmp\nmv /etc/origin/ovn/ovnsb_db.db /tmp\n</code></pre>"},{"location":"en/reference/document-convention/#link","title":"Link","text":"<p>Links in the site use the corresponding <code>md</code> file path.</p> BadGood <pre><code>Please refer to [Preparation](http://kubeovn.github.io/prepare) before installation.\n</code></pre> <pre><code>Please refer to [Preparation](./prepare.md) before installation.\n</code></pre> BadGood <pre><code>If you have any questions, please refer to [Kubernetes Documentation](http://kubernetes.io).\n</code></pre> <pre><code>If you have any questions, please refer to [Kubernetes Documentation](http://kubernetes.io){: target=\"_blank\" }.\n</code></pre>"},{"location":"en/reference/document-convention/#empty-line","title":"Empty Line","text":"<p>Different logical blocks, such as title and text, text and code, text and number need to be separated by blank lines.</p> BadGood <pre><code>Download the script below to install it:\n```bash\nwget 127.0.0.1\n```\n</code></pre> <pre><code>Download the script below to install it:\n\n```bash\nwget 127.0.0.1\n```\n</code></pre> <p>Separate logical blocks with only one blank line.</p> BadGood <pre><code>Download the script below to install it:\n\n\n\n```bash\nwget 127.0.0.1\n```\n</code></pre> <pre><code>Download the script below to install it:\n\n```bash\nwget 127.0.0.1\n```\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/reference/feature-stage/","title":"Feature Stage","text":"<p>In Kube-OVN, feature stage is classified into Alpha, Beta and GA, based on the degree of feature usage, documentation and test coverage.</p>"},{"location":"en/reference/feature-stage/#definition-of-stage","title":"Definition of Stage","text":"<p>For Alpha stage functions:</p> <ul> <li>The feature is not fully documented and well tested.</li> <li>This feature may change or even be removed in the future.</li> <li>This feature API is not guaranteed to be stable and may be removed.</li> <li>Community provides low priority support for this feature and long-term support cannot be guaranteed.</li> <li>Since feature stability and long-term support cannot be guaranteed, it can be tested and verified, but is not recommended for production use.</li> </ul> <p>For Beta stage functions:</p> <ul> <li>This feature is partially documented and tested, but complete coverage is not guaranteed.</li> <li>This feature may change in the future and the upgrade may affect the network, but it will not be removed as a whole.</li> <li>This feature API may change in the future and the fields may be adjusted, but not removed as a whole.</li> <li>This feature will be supported by the community in the long term.</li> <li>It can be used on non-critical services as the functionality will be supported for a long time, but it is not recommended for critical production service as there is a possibility of changes in functionality and APIs that may break the network.</li> </ul> <p>For GA stage functions:</p> <ul> <li>The feature has full documentation and test coverage.</li> <li>The feature will remain stable and upgrades will be guaranteed to be smooth.</li> <li>This feature API is not subject to disruptive changes.</li> <li>This feature will be supported with high priority by the community and long-term support will be guaranteed.</li> </ul>"},{"location":"en/reference/feature-stage/#feature-stage-list","title":"Feature Stage List","text":"<p>This list records the feature stages from the 1.8 release.</p> Feature Default Stage Since Until Namespaced Subnet true GA 1.8 Distributed Gateway true GA 1.8 Active-backup Centralized Gateway true GA 1.8 ECMP Centralized Gateway false Beta 1.8 Subnet ACL true Alpha 1.9 Subnet Isolation (Will be replaced by ACL later) true Beta 1.8 Underlay Subnet true GA 1.8 Multiple Pod Interface true Beta 1.8 Subnet DHCP false Alpha 1.10 Subnet with External Gateway false Alpha 1.8 Cluster Inter-Connection with OVN-IC false Beta 1.8 Cluster Inter-Connection with Submariner false Alpha 1.9 VIP Reservation true Alpha 1.10 Create Custom VPC true Beta 1.8 Custom VPC Floating IP/SNAT/DNAT true Alpha 1.10 Custom VPC Static Route true Alpha 1.10 Custom VPC Policy Route true Alpha 1.10 Custom VPC Security Group true Alpha 1.10 Container Bandwidth QoS true GA 1.8 linux-netem QoS true Alpha 1.9 Prometheus Integration false GA 1.8 Grafana Integration false GA 1.8 IPv4/v6 DualStack false GA 1.8 Default VPC EIP/SNAT false Beta 1.8 Traffic Mirroring false GA 1.8 NetworkPolicy true Beta 1.8 Webhook false Alpha 1.10 Performance Tunning false Beta 1.8 Interconnection with Routes in Overlay Mode false Alpha 1.8 BGP Support false Alpha 1.9 Cilium Integration false Alpha 1.10 Custom VPC Peering false Alpha 1.10 Mellanox Offload false Alpha 1.8 Corigine Offload false Alpha 1.10 Windows Support false Alpha 1.10 DPDK Support false Alpha 1.10 OpenStack Integration false Alpha 1.9 Single Pod Fixed IP/Mac true GA 1.8 Workload with Fixed IP true GA 1.8 StatefulSet with Fixed IP true GA 1.8 VM with Fixed IP false Beta 1.9 Load Balancer Type Service in Default VPC false Alpha 1.11 Load Balance in Custom VPC false Alpha 1.11 DNS in Custom VPC false Alpha 1.11 Underlay and Overlay Interconnection false Alpha 1.11 <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/reference/iptables-rules/","title":"Iptables Rules","text":"<p>Kube-OVN uses <code>ipset</code> and <code>iptables</code> to implement gateway NAT functionality in the default VPC overlay Subnets.</p> <p>The ipset used is shown in the following table:</p> Name\uff08IPv4/IPv6\uff09 Type Usage ovn40services/ovn60services hash:net Service CIDR ovn40subnets/ovn60subnets hash:net Overlay Subnet CIDR and NodeLocal DNS IP address ovn40subnets-nat/ovn60subnets-nat hash:net Overlay Subnet CIDRs that enable <code>NatOutgoing</code> ovn40subnets-distributed-gw/ovn60subnets-distributed-gw hash:net Overlay Subnet CIDRs that use distributed gateway ovn40other-node/ovn60other-node hash:net Internal IP addresses for other Nodes ovn40local-pod-ip-nat/ovn60local-pod-ip-nat hash:ip Deprecated ovn40subnets-nat-policy hash:net All subnet cidrs configured with natOutgoingPolicyRules ovn40natpr-418e79269dc5-dst hash:net The dstIPs corresponding to the rule in natOutgoingPolicyRules ovn40natpr-418e79269dc5-src hash:net The srcIPs corresponding to the rule in natOutgoingPolicyRules <p>The iptables rules (IPv4) used are shown in the following table:</p> Table Chain Rule Usage Note filter INPUT -m set --match-set ovn40services src -j ACCEPT Allow k8s service and pod traffic to pass through -- filter INPUT -m set --match-set ovn40services dst -j ACCEPT Allow k8s service and pod traffic to pass through -- filter INPUT -m set --match-set ovn40subnets src -j ACCEPT Allow k8s service and pod traffic to pass through -- filter INPUT -m set --match-set ovn40subnets dst -j ACCEPT Allow k8s service and pod traffic to pass through -- filter FORWARD -m set --match-set ovn40services src -j ACCEPT Allow k8s service and pod traffic to pass through -- filter FORWARD -m set --match-set ovn40services dst -j ACCEPT Allow k8s service and pod traffic to pass through -- filter FORWARD -m set --match-set ovn40subnets src -j ACCEPT Allow k8s service and pod traffic to pass through -- filter FORWARD -m set --match-set ovn40subnets dst -j ACCEPT Allow k8s service and pod traffic to pass through -- filter FORWARD -s 10.16.0.0/16 -m comment --comment \"ovn-subnet-gateway,ovn-default\" Used to count packets from the subnet to the external network \"10.16.0.0/16\" is the cidr of the subnet, the \"ovn-subnet-gateway\" before the \",\" in comment is used to identify the iptables rule used to count the subnet inbound and outbound gateway packets, and the \"ovn-default\" after the \",\" is the name of the subnet filter FORWARD -d 10.16.0.0/16 -m comment --comment \"ovn-subnet-gateway,ovn-default\" Used to count packets from the external network accessing the subnet \"10.16.0.0/16\" is the cidr of the subnet, the \"ovn-subnet-gateway\" before the \",\" in comment is used to identify the iptables rule used to count the subnet inbound and outbound gateway packets, and the \"ovn-default\" after the \",\" is the name of the subnet filter OUTPUT -p udp -m udp --dport 6081 -j MARK --set-xmark 0x0 Clear traffic tag to prevent SNAT UDP: bad checksum on VXLAN interface nat PREROUTING -m comment --comment \"kube-ovn prerouting rules\" -j OVN-PREROUTING Enter OVN-PREROUTING chain processing -- nat POSTROUTING -m comment --comment \"kube-ovn postrouting rules\" -j OVN-POSTROUTING Enter OVN-POSTROUTING chain processing -- nat OVN-PREROUTING -i ovn0 -m set --match-set ovn40subnets src -m set --match-set ovn40services dst -j MARK --set-xmark 0x4000/0x4000 Adding masquerade tags to Pod access service traffic Used when the built-in LB is turned off nat OVN-PREROUTING -p tcp -m addrtype --dst-type LOCAL -m set --match-set KUBE-NODE-PORT-LOCAL-TCP dst -j MARK --set-xmark 0x80000/0x80000 Add specific tags to ExternalTrafficPolicy for Local's Service traffic (TCP) Only used when kube-proxy is using ipvs mode nat OVN-PREROUTING -p udp -m addrtype --dst-type LOCAL -m set --match-set KUBE-NODE-PORT-LOCAL-UDP dst -j MARK --set-xmark 0x80000/0x80000 Add specific tags to ExternalTrafficPolicy for Local's Service traffic (UDP) Only used when kube-proxy is using ipvs mode nat OVN-POSTROUTING -m set --match-set ovn40services src -m set --match-set ovn40subnets dst -m mark --mark 0x4000/0x4000 -j SNAT --to-source  Use node IP as the source address for access from node to overlay Pods via service IP\u3002 Works only when kube-proxy is using ipvs mode nat OVN-POSTROUTING -m mark --mark 0x4000/0x4000 -j MASQUERADE Perform SNAT for specific tagged traffic -- nat OVN-POSTROUTING -m set --match-set ovn40subnets src -m set --match-set ovn40subnets dst -j MASQUERADE Perform SNAT for Service traffic between Pods passing through the node -- nat OVN-POSTROUTING -m mark --mark 0x80000/0x80000 -m set --match-set ovn40subnets-distributed-gw dst -j RETURN For Service traffic where ExternalTrafficPolicy is Local, if the Endpoint uses a distributed gateway, SNAT is not required. -- nat OVN-POSTROUTING -m mark --mark 0x80000/0x80000 -j MASQUERADE For Service traffic where ExternalTrafficPolicy is Local, if the Endpoint uses a centralized gateway, SNAT is required. -- nat OVN-POSTROUTING -p tcp -m tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN No SNAT is performed when the Pod IP is exposed to the outside world -- nat OVN-POSTROUTING -s 10.16.0.0/16 -m set ! --match-set ovn40subnets dst -j SNAT --to-source 192.168.0.101 When the Pod accesses the network outside the cluster, if the subnet is NatOutgoing and a centralized gateway with the specified IP is used, perform SNAT 10.16.0.0/16 is the Subnet CIDR\uff0c192.168.0.101 is the specified IP of gateway node nat OVN-POSTROUTING -m set --match-set ovn40subnets-nat src -m set ! --match-set ovn40subnets dst -j MASQUERADE When the Pod accesses the network outside the cluster, if NatOutgoing is enabled on the subnet, perform SNAT -- nat OVN-POSTROUTING -m set --match-set ovn40subnets-nat-policy src -m set ! --match-set ovn40subnets dst -j OVN-NAT-POLICY When Pod accesses the network outside the cluster, if natOutgoingPolicyRules is enabled on the subnet, the packet with the specified policy will perform SNAT ovn40subnets-nat-policy is all subnet segments configured with natOutgoingPolicyRules nat OVN-POSTROUTING -m mark --mark 0x90001/0x90001 -j MASQUERADE --random-fully When Pod accesses the network outside the cluster, if natOutgoingPolicyRules is enabled on the subnet, the packet with the specified policy will perform SNAT After coming out of OVN-NAT-POLICY, if it is tagged with 0x90001/0x90001, it will do SNAT nat OVN-POSTROUTING -m mark --mark 0x90002/0x90002 -j RETURN When Pod accesses the network outside the cluster, if natOutgoingPolicyRules is enabled on the subnet, the packet with the specified policy will perform SNAT After coming out of OVN-NAT-POLICY, if it is tagged with 0x90002/0x90002, it will not do SNAT nat OVN-NAT-POLICY -s 10.0.11.0/24 -m comment --comment natPolicySubnet-net1 -j OVN-NAT-PSUBNET-aa98851157c5 When Pod accesses the network outside the cluster, if natOutgoingPolicyRules is enabled on the subnet, the packet with the specified policy will perform SNAT 10.0.11.0/24 represents the CIDR of the subnet net1, and the rules under the OVN-NAT-PSUBNET-aa98851157c5 chain correspond to the natOutgoingPolicyRules configuration of this subnet nat OVN-NAT-PSUBNET-xxxxxxxxxxxx -m set --match-set ovn40natpr-418e79269dc5-src src -m set --match-set ovn40natpr-418e79269dc5-dst dst -j MARK --set-xmark 0x90002/0x90002 When Pod accesses the network outside the cluster, if natOutgoingPolicyRules is enabled on the subnet, the packet with the specified policy will perform SNAT 418e79269dc5 indicates the ID of a rule in natOutgoingPolicyRules, which can be viewed through status.natOutgoingPolicyRules[index].RuleID, indicating that srcIPs meets ovn40natpr-418e79269dc5-src, and dstIPS meets ovn40natpr-418e79269dc5- dst will be marked with tag 0x90002 mangle OVN-OUTPUT -d 10.241.39.2/32 -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x90003/0x90003 Introduce kubelet's detection traffic to tproxy with a specific mark mangle OVN-PREROUTING -d 10.241.39.2/32 -p tcp -m tcp --dport 80 -j TPROXY --on-port 8102 --on-ip 172.18.0.3 --tproxy-mark 0x90004/0x90004 Introduce kubelet's detection traffic to tproxy with a specific mark <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/reference/kube-ovn-api/","title":"Kube-OVN API Reference","text":"<p>Based on Kube-OVN v1.12.0, we have compiled a list of CRD resources supported by Kube-OVN, listing the types and meanings of each field of CRD definition for reference.</p>"},{"location":"en/reference/kube-ovn-api/#generic-condition-definition","title":"Generic Condition Definition","text":"Property Name Type Description type String Type of status status String The value of status, in the range of <code>True</code>, <code>False</code> or <code>Unknown</code> reason String The reason for the status change message String The specific message of the status change lastUpdateTime Time The last time the status was updated lastTransitionTime Time Time of last status type change <p>In each CRD definition, the Condition field in Status follows the above format, so we explain it in advance.</p>"},{"location":"en/reference/kube-ovn-api/#subnet-definition","title":"Subnet Definition","text":""},{"location":"en/reference/kube-ovn-api/#subnet","title":"Subnet","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources have this value as kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource will have the value <code>Subnet</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec SubnetSpec Subnet specific configuration information status SubnetStatus Subnet status information"},{"location":"en/reference/kube-ovn-api/#subnetspec","title":"SubnetSpec","text":"Property Name Type Description default Bool Whether this subnet is the default subnet vpc String The vpc which the subnet belongs to, default is ovn-cluster protocol String IP protocol, the value is in the range of <code>IPv4</code>, <code>IPv6</code> or <code>Dual</code> namespaces []String The list of namespaces bound to this subnet cidrBlock String The range of the subnet, e.g. 10.16.0.0/16 gateway String The gateway address of the subnet, the default value is the first available address under the CIDRBlock of the subnet excludeIps []String The range of addresses under this subnet that will not be automatically assigned provider String Default value is <code>ovn</code>. In the case of multiple NICs, the value is <code>&lt;name&gt;.&lt;namespace&gt;</code> of the NetworkAttachmentDefinition, Kube-OVN will use this information to find the corresponding subnet resource gatewayType String The gateway type in overlay mode, either <code>distributed</code> or <code>centralized</code> gatewayNode String The gateway node when the gateway mode is centralized, node names can be comma-separated natOutgoing Bool Whether the outgoing traffic is NAT externalEgressGateway String The address of the external gateway. This parameter and the natOutgoing parameter cannot be set at the same time policyRoutingPriority Uint32 Policy route priority. Used to control the forwarding of traffic to the external gateway address after the subnet gateway policyRoutingTableID Uint32 The TableID of the local policy routing table, should be different for each subnet to avoid conflicts private Bool Whether the subnet is a private subnet, which denies access to addresses inside the subnet if the subnet is private allowSubnets []String If the subnet is a private subnet, the set of addresses that are allowed to access the subnet vlan String The name of vlan to which the subnet is bound vips []String The virtual-ip parameter information for virtual type lsp on the subnet logicalGateway Bool Whether to enable logical gateway disableGatewayCheck Bool Whether to skip the gateway connectivity check when creating a pod disableInterConnection Bool Whether to enable subnet interconnection across clusters enableDHCP Bool Whether to configure dhcp configuration options for lsps belong this subnet dhcpV4Options String The DHCP_Options record associated with lsp dhcpv4_options on the subnet dhcpV6Options String The DHCP_Options record associated with lsp dhcpv6_options on the subnet enableIPv6RA Bool Whether to configure the ipv6_ra_configs parameter for the lrp port of the router connected to the subnet ipv6RAConfigs String The ipv6_ra_configs parameter configuration for the lrp port of the router connected to the subnet acls []Acl The acls record associated with the logical-switch of the subnet u2oInterconnection Bool Whether to enable interconnection mode for Overlay/Underlay enableLb *Bool Whether the logical-switch of the subnet is associated with load-balancer records enableEcmp Bool Centralized subnet, whether to enable ECMP routing"},{"location":"en/reference/kube-ovn-api/#acl","title":"Acl","text":"Property Name Type Description direction String Restrict the direction of acl, which value is <code>from-lport</code> or <code>to-lport</code> priority Int Acl priority, in the range 0 to 32767 match String Acl rule match expression action String The action of the rule, which value is in the range of <code>allow-related</code>, <code>allow-stateless</code>, <code>allow</code>, <code>drop</code>, <code>reject</code>"},{"location":"en/reference/kube-ovn-api/#subnetstatus","title":"SubnetStatus","text":"Property Name Type Description conditions []SubnetCondition Subnet status change information, refer to the beginning of the document for the definition of Condition v4AvailableIPs Float64 Number of available IPv4 IPs v4availableIPrange String The available range of IPv4 addresses on the subnet v4UsingIPs Float64 Number of used IPv4 IPs v4usingIPrange String Used IPv4 address ranges on the subnet v6AvailableIPs Float64 Number of available IPv6 IPs v6availableIPrange String The available range of IPv6 addresses on the subnet v6UsingIPs Float64 Number of used IPv6 IPs v6usingIPrange String Used IPv6 address ranges on the subnet sctivateGateway String The currently working gateway node in centralized subnet of master-backup mode dhcpV4OptionsUUID String The DHCP_Options record identifier associated with the lsp dhcpv4_options on the subnet dhcpV6OptionsUUID String The DHCP_Options record identifier associated with the lsp dhcpv6_options on the subnet u2oInterconnectionIP String The IP address used for interconnection when Overlay/Underlay interconnection mode is enabled"},{"location":"en/reference/kube-ovn-api/#ip-definition","title":"IP Definition","text":""},{"location":"en/reference/kube-ovn-api/#ip","title":"IP","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources are kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource have the value <code>IP</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec IPSpec IP specific configuration information"},{"location":"en/reference/kube-ovn-api/#ipsepc","title":"IPSepc","text":"Property Name Type Description podName String Pod name which assigned with this IP namespace String The name of the namespace where the pod is bound subnet String The subnet which the ip belongs to attachSubnets []String The name of the other subnets attached to this primary IP (field deprecated) nodeName String The name of the node where the pod is bound ipAddress String IP address, in <code>v4IP,v6IP</code> format for dual-stack cases v4IPAddress String IPv4 IP address v6IPAddress String IPv6 IP address attachIPs []String Other IP addresses attached to this primary IP (field is deprecated) macAddress String The Mac address of the bound pod attachMacs []String Other Mac addresses attached to this primary IP (field deprecated) containerID String The Container ID corresponding to the bound pod podType String Special workload pod, can be <code>StatefulSet</code>, <code>VirtualMachine</code> or empty"},{"location":"en/reference/kube-ovn-api/#underlay-configuration","title":"Underlay configuration","text":""},{"location":"en/reference/kube-ovn-api/#vlan","title":"Vlan","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all instances of this resource will be kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource will have the value <code>Vlan</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec VlanSpec Vlan specific configuration information status VlanStatus Vlan status information"},{"location":"en/reference/kube-ovn-api/#vlanspec","title":"VlanSpec","text":"Property Name Type Description id Int Vlan tag number, in the range of 0~4096 provider String The name of the ProviderNetwork to which the vlan is bound"},{"location":"en/reference/kube-ovn-api/#vlanstatus","title":"VlanStatus","text":"Property Name Type Description subnets []String The list of subnets to which the vlan is bound conditions []VlanCondition Vlan status change information, refer to the beginning of the document for the definition of Condition"},{"location":"en/reference/kube-ovn-api/#providernetwork","title":"ProviderNetwork","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources are kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource will have the value <code>ProviderNetwork</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec ProviderNetworkSpec ProviderNetwork specific configuration information status ProviderNetworkStatus ProviderNetwork status information"},{"location":"en/reference/kube-ovn-api/#providernetworkspec","title":"ProviderNetworkSpec","text":"Property Name Type Description defaultInterface String The name of the NIC interface used by default for this bridge network customInterfaces []CustomInterface The special NIC configuration used by this bridge network excludeNodes []String The names of the nodes that will not be bound to this bridge network exchangeLinkName Bool Whether to exchange the bridge NIC and the corresponding OVS bridge name"},{"location":"en/reference/kube-ovn-api/#custominterface","title":"CustomInterface","text":"Property Name Type Description interface String NIC interface name used for underlay nodes []String List of nodes using the custom NIC interface"},{"location":"en/reference/kube-ovn-api/#providernetworkstatus","title":"ProviderNetworkStatus","text":"Property Name Type Description ready Bool Whether the current bridge network is in the ready state readyNodes []String The name of the node whose bridge network is ready notReadyNodes []String The name of the node whose bridge network is not ready vlans []String The name of the vlan to which the bridge network is bound conditions []ProviderNetworkCondition ProviderNetwork status change information, refer to the beginning of the document for the definition of Condition"},{"location":"en/reference/kube-ovn-api/#vpc-definition","title":"Vpc Definition","text":""},{"location":"en/reference/kube-ovn-api/#vpc","title":"Vpc","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources have this value as kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource will have the value <code>Vpc</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec VpcSpec Vpc specific configuration information status VpcStatus Vpc status information"},{"location":"en/reference/kube-ovn-api/#vpcspec","title":"VpcSpec","text":"Property Name Type Description namespaces []String List of namespaces bound by Vpc staticRoutes []*StaticRoute The static route information configured under Vpc policyRoutes []*PolicyRoute The policy route information configured under Vpc vpcPeerings []*VpcPeering Vpc interconnection information enableExternal Bool Whether vpc is connected to an external switch"},{"location":"en/reference/kube-ovn-api/#staticroute","title":"StaticRoute","text":"Property Name Type Description policy String Routing policy, takes the value of <code>policySrc</code> or <code>policyDst</code> cidr String Routing cidr value nextHopIP String The next hop information of the route"},{"location":"en/reference/kube-ovn-api/#policyroute","title":"PolicyRoute","text":"Property Name Type Description priority Int32 Priority for policy route match String Match expression for policy route action String Action for policy route, the value is in the range of <code>allow</code>, <code>drop</code>, <code>reroute</code> nextHopIP String The next hop of the policy route, separated by commas in the case of ECMP routing"},{"location":"en/reference/kube-ovn-api/#vpcpeering","title":"VpcPeering","text":"Property Name Type Description remoteVpc String Name of the interconnected peering vpc localConnectIP String The local ip for vpc used to connect to peer vpc"},{"location":"en/reference/kube-ovn-api/#vpcstatus","title":"VpcStatus","text":"Property Name Type Description conditions []VpcCondition Vpc status change information, refer to the beginning of the documentation for the definition of Condition standby Bool Whether the vpc creation is complete, the subnet under the vpc needs to wait for the vpc creation to complete other proceeding default Bool Whether it is the default vpc defaultLogicalSwitch String The default subnet under vpc router String The logical-router name for the vpc tcpLoadBalancer String TCP LB information for vpc udpLoadBalancer String UDP LB information for vpc tcpSessionLoadBalancer String TCP Session Hold LB Information for Vpc udpSessionLoadBalancer String UDP session hold LB information for Vpc subnets []String List of subnets for vpc vpcPeerings []String List of peer vpcs for vpc interconnection enableExternal Bool Whether the vpc is connected to an external switch"},{"location":"en/reference/kube-ovn-api/#vpcnatgateway","title":"VpcNatGateway","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources are kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource will have the value <code>VpcNatGateway</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec VpcNatSpec Vpc gateway specific configuration information"},{"location":"en/reference/kube-ovn-api/#vpcnatspec","title":"VpcNatSpec","text":"Property Name Type Description vpc String Vpc name which the vpc gateway belongs to subnet String The name of the subnet to which the gateway pod belongs lanIp String The IP address assigned to the gateway pod selector []String Standard Kubernetes selector match information tolerations []VpcNatToleration Standard Kubernetes tolerance information"},{"location":"en/reference/kube-ovn-api/#vpcnattoleration","title":"VpcNatToleration","text":"Property Name Type Description key String The key information of the taint tolerance operator String Takes the value of <code>Exists</code> or <code>Equal</code> value String The value information of the taint tolerance effect String The effect of the taint tolerance, takes the value of <code>NoExecute</code>, <code>NoSchedule</code>, or <code>PreferNoSchedule</code> tolerationSeconds Int64 The amount of time the pod can continue to run on the node after the taint is added <p>The meaning of the above tolerance fields can be found in the official Kubernetes documentation Taint and Tolerance.</p>"},{"location":"en/reference/kube-ovn-api/#iptableseip","title":"IptablesEIP","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources have this value as kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource have the value <code>IptablesEIP</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec IptablesEipSpec IptablesEIP specific configuration information used by vpc gateway status IptablesEipStatus IptablesEIP status information used by vpc gateway"},{"location":"en/reference/kube-ovn-api/#iptableseipspec","title":"IptablesEipSpec","text":"Property Name Type Description v4ip String IptablesEIP v4 address v6ip String IptablesEIP v6 address macAddress String The assigned mac address, not actually used natGwDp String Vpc gateway name"},{"location":"en/reference/kube-ovn-api/#iptableseipstatus","title":"IptablesEipStatus","text":"Property Name Type Description ready Bool Whether IptablesEIP is configured complete ip String The IP address used by IptablesEIP, currently only IPv4 addresses are supported redo String IptablesEIP crd creation or update time nat String The type of IptablesEIP, either <code>fip</code>, <code>snat</code>, or <code>dnat</code> conditions []IptablesEIPCondition IptablesEIP status change information, refer to the beginning of the documentation for the definition of Condition"},{"location":"en/reference/kube-ovn-api/#iptablesfiprule","title":"IptablesFIPRule","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources have this value as kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource have the value <code>IptablesFIPRule</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec IptablesFIPRuleSpec The IptablesFIPRule specific configuration information used by vpc gateway status IptablesFIPRuleStatus IptablesFIPRule status information used by vpc gateway"},{"location":"en/reference/kube-ovn-api/#iptablesfiprulespec","title":"IptablesFIPRuleSpec","text":"Property Name Type Description eip String Name of the IptablesEIP used for IptablesFIPRule internalIp String The corresponding internal IP address"},{"location":"en/reference/kube-ovn-api/#iptablesfiprulestatus","title":"IptablesFIPRuleStatus","text":"Property Name Type Description ready Bool Whether IptablesFIPRule is configured or not v4ip String The v4 IP address used by IptablesEIP v6ip String The v6 IP address used by IptablesEIP natGwDp String Vpc gateway name redo String IptablesFIPRule crd creation or update time conditions []IptablesFIPRuleCondition IptablesFIPRule status change information, refer to the beginning of the documentation for the definition of Condition"},{"location":"en/reference/kube-ovn-api/#iptablessnatrule","title":"IptablesSnatRule","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources have this value as kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource have the value <code>IptablesSnatRule</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec IptablesSnatRuleSpec The IptablesSnatRule specific configuration information used by the vpc gateway status IptablesSnatRuleStatus IptablesSnatRule status information used by vpc gateway"},{"location":"en/reference/kube-ovn-api/#iptablessnatrulespec","title":"IptablesSnatRuleSpec","text":"Property Name Type Description eip String Name of the IptablesEIP used by IptablesSnatRule internalIp String IptablesSnatRule's corresponding internal IP address"},{"location":"en/reference/kube-ovn-api/#iptablessnatrulestatus","title":"IptablesSnatRuleStatus","text":"Property Name Type Description ready Bool Whether the configuration is complete v4ip String The v4 IP address used by IptablesSnatRule v6ip String The v6 IP address used by IptablesSnatRule natGwDp String Vpc gateway name redo String IptablesSnatRule crd creation or update time conditions []IptablesSnatRuleCondition IptablesSnatRule status change information, refer to the beginning of the documentation for the definition of Condition"},{"location":"en/reference/kube-ovn-api/#iptablesdnatrule","title":"IptablesDnatRule","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources have this value as kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource have the value <code>IptablesDnatRule</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec IptablesDnatRuleSpec The IptablesDnatRule specific configuration information used by vpc gateway status IptablesDnatRuleStatus IptablesDnatRule status information used by vpc gateway"},{"location":"en/reference/kube-ovn-api/#iptablesdnatrulespec","title":"IptablesDnatRuleSpec","text":"Property Name Type Description eip Sting Name of IptablesEIP used by IptablesDnatRule externalPort Sting External port used by IptablesDnatRule protocol Sting Vpc gateway dnat protocol type internalIp Sting Internal IP address used by IptablesDnatRule internalPort Sting Internal port used by IptablesDnatRule"},{"location":"en/reference/kube-ovn-api/#iptablesdnatrulestatus","title":"IptablesDnatRuleStatus","text":"Property Name Type Description ready Bool Whether the configuration is complete v4ip String The v4 IP address used by IptablesDnatRule v6ip String The v6 IP address used by IptablesDnatRule natGwDp String Vpc gateway name redo String IptablesDnatRule crd creation or update time conditions []IptablesDnatRuleCondition IptablesDnatRule Status change information, refer to the beginning of the documentation for the definition of Condition"},{"location":"en/reference/kube-ovn-api/#vpcdns","title":"VpcDns","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources have kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource will have the value <code>VpcDns</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec VpcDnsSpec VpcDns specific configuration information status VpcDnsStatus VpcDns status information"},{"location":"en/reference/kube-ovn-api/#vpcdnsspec","title":"VpcDnsSpec","text":"Property Name Type Description vpc String Name of the vpc where VpcDns is located subnet String The subnet name of the address assigned to the VpcDns pod"},{"location":"en/reference/kube-ovn-api/#vpcdnsstatus","title":"VpcDnsStatus","text":"Property Name Type Description conditions []VpcDnsCondition VpcDns status change information, refer to the beginning of the document for the definition of Condition active Bool Whether VpcDns is in use <p>For detailed documentation on the use of VpcDns, see Customizing VPC DNS.</p>"},{"location":"en/reference/kube-ovn-api/#switchlbrule","title":"SwitchLBRule","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources have this value as kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource will have the value <code>SwitchLBRule</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec SwitchLBRuleSpec SwitchLBRule specific configuration information status SwitchLBRuleStatus SwitchLBRule status information"},{"location":"en/reference/kube-ovn-api/#switchlbrulespec","title":"SwitchLBRuleSpec","text":"Property Name Type Description vip String Vip address of SwitchLBRule namespace String SwitchLBRule's namespace selector []String Standard Kubernetes selector match information sessionAffinity String Standard Kubernetes service sessionAffinity value ports []SlrPort List of SwitchLBRule ports <p>For detailed configuration information of SwitchLBRule, you can refer to Customizing VPC Internal Load Balancing.</p>"},{"location":"en/reference/kube-ovn-api/#slrport","title":"SlrPort","text":"Property Name Type Description name String Port name port Int32 Port number targetPort Int32 Target port of SwitchLBRule protocol String Protocol type"},{"location":"en/reference/kube-ovn-api/#switchlbrulestatus","title":"SwitchLBRuleStatus","text":"Property Name Type Description conditions []SwitchLBRuleCondition SwitchLBRule status change information, refer to the beginning of the document for the definition of Condition ports String Port information service String Name of the service"},{"location":"en/reference/kube-ovn-api/#security-group-and-vip","title":"Security Group and Vip","text":""},{"location":"en/reference/kube-ovn-api/#securitygroup","title":"SecurityGroup","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources are kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource will have a value of <code>SecurityGroup</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec SecurityGroupSpec Security Group specific configuration information status SecurityGroupStatus Security group status information"},{"location":"en/reference/kube-ovn-api/#securitygroupspec","title":"SecurityGroupSpec","text":"Property Name Type Description ingressRules []*SgRule Inbound security group rules egressRules []*SgRule Outbound security group rules allowSameGroupTraffic Bool Whether lsps in the same security group can interoperate and whether traffic rules need to be updated"},{"location":"en/reference/kube-ovn-api/#sgrule","title":"SgRule","text":"Property Name Type Description ipVersion String IP version number, <code>ipv4</code> or <code>ipv6</code> protocol String The value of <code>icmp</code>, <code>tcp</code>, or <code>udp</code> priority Int Acl priority. The value range is 1-200, the smaller the value, the higher the priority. remoteType String The value is either <code>address</code> or <code>securityGroup</code> remoteAddress String The address of the other side remoteSecurityGroup String The name of security group on the other side portRangeMin Int The starting value of the port range, the minimum value is 1. portRangeMax Int The ending value of the port range, the maximum value is 65535. policy String The value is <code>allow</code> or <code>drop</code>"},{"location":"en/reference/kube-ovn-api/#securitygroupstatus","title":"SecurityGroupStatus","text":"Property Name Type Description portGroup String The name of the port-group for the security group allowSameGroupTraffic Bool Whether lsps in the same security group can interoperate, and whether the security group traffic rules need to be updated ingressMd5 String The MD5 value of the inbound security group rule egressMd5 String The MD5 value of the outbound security group rule ingressLastSyncSuccess Bool Whether the last synchronization of the inbound rule was successful egressLastSyncSuccess Bool Whether the last synchronization of the outbound rule was successful"},{"location":"en/reference/kube-ovn-api/#vip","title":"Vip","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources are kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource will have the value <code>Vip</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec VipSpec Vip specific configuration information status VipStatus Vip status information"},{"location":"en/reference/kube-ovn-api/#vipspec","title":"VipSpec","text":"Property Name Type Description namespace String Vip's namespace subnet String Vip's subnet v4ip String Vip IPv4 ip address v6ip String Vip IPv6 ip address macAddress String Vip mac address parentV4ip String Not currently in use parentV6ip String Not currently in use parentMac String Not currently in use attachSubnets []String This field is deprecated and no longer used"},{"location":"en/reference/kube-ovn-api/#vipstatus","title":"VipStatus","text":"Property Name Type Description conditions []VipCondition Vip status change information, refer to the beginning of the documentation for the definition of Condition ready Bool Vip is ready or not v4ip String Vip IPv4 ip address, should be the same as the spec field v6ip String Vip IPv6 ip address, should be the same as the spec field mac String The vip mac address, which should be the same as the spec field pv4ip String Not currently used pv6ip String Not currently used pmac String Not currently used"},{"location":"en/reference/kube-ovn-api/#ovneip","title":"OvnEip","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources are kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource will have the value <code>OvnEip</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec OvnEipSpec OvnEip specific configuration information for default vpc status OvnEipStatus OvnEip status information for default vpc"},{"location":"en/reference/kube-ovn-api/#ovneipspec","title":"OvnEipSpec","text":"Property Name Type Description externalSubnet String OvnEip's subnet name v4ip String OvnEip IP address macAddress String OvnEip Mac address type String OvnEip use type, the value can be <code>fip</code>, <code>snat</code> or <code>lrp</code>"},{"location":"en/reference/kube-ovn-api/#ovneipstatus","title":"OvnEipStatus","text":"Property Name Type Description conditions []OvnEipCondition OvnEip status change information, refer to the beginning of the documentation for the definition of Condition v4ip String The IPv4 ip address used by ovnEip macAddress String Mac address used by ovnEip"},{"location":"en/reference/kube-ovn-api/#ovnfip","title":"OvnFip","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources are kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource will have the value <code>OvnFip</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec OvnFipSpec OvnFip specific configuration information in default vpc status OvnFipStatus OvnFip status information in default vpc"},{"location":"en/reference/kube-ovn-api/#ovnfipspec","title":"OvnFipSpec","text":"Property Name Type Description ovnEip String Name of the bound ovnEip ipName String The IP crd name corresponding to the bound Pod"},{"location":"en/reference/kube-ovn-api/#ovnfipstatus","title":"OvnFipStatus","text":"Property Name Type Description ready Bool OvnFip is ready or not v4Eip String Name of the ovnEip to which ovnFip is bound v4Ip String The ovnEip address currently in use macAddress String OvnFip's configured mac address vpc String The name of the vpc where ovnFip is located conditions []OvnFipCondition OvnFip status change information, refer to the beginning of the document for the definition of Condition"},{"location":"en/reference/kube-ovn-api/#ovnsnatrule","title":"OvnSnatRule","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources have kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource will have the value <code>OvnSnatRule</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec OvnSnatRuleSpec OvnSnatRule specific configuration information in default vpc status OvnSnatRuleStatus OvnSnatRule status information in default vpc"},{"location":"en/reference/kube-ovn-api/#ovnsnatrulespec","title":"OvnSnatRuleSpec","text":"Property Name Type Description ovnEip String Name of the ovnEip to which ovnSnatRule is bound vpcSubnet String The name of the subnet configured by ovnSnatRule ipName String The IP crd name corresponding to the ovnSnatRule bound Pod"},{"location":"en/reference/kube-ovn-api/#ovnsnatrulestatus","title":"OvnSnatRuleStatus","text":"Property Name Type Description ready Bool OvnSnatRule is ready or not v4Eip String The ovnEip address to which ovnSnatRule is bound v4IpCidr String The cidr address used to configure snat in the logical-router vpc String The name of the vpc where ovnSnatRule is located conditions []OvnSnatRuleCondition OvnSnatRule status change information, refer to the beginning of the document for the definition of Condition <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/reference/kube-ovn-pinger-args/","title":"Kube-OVN-Pinger args Reference","text":"<p>Based on the Kube-OVN v1.12.0 version, We have compiled the parameters supported by Kube-ovn-pinger, and listed the value types, meanings, and default values of each field defined by the parameters for reference</p>"},{"location":"en/reference/kube-ovn-pinger-args/#args-describeption","title":"Args Describeption","text":"Arg Name Type Description Default Value port Int metrics port 8080 kubeconfig String Path to kubeconfig file with authorization and master location information. If not set use the inCluster token. \"\" ds-namespace String kube-ovn-pinger daemonset namespace \"kube-system\" ds-name String kube-ovn-pinger daemonset name \"kube-ovn-pinger\" interval Int interval seconds between consecutive pings 5 mode String server or job Mode \"server\" exit-code Int exit code when failure happens 0 internal-dns String check dns from pod \"kubernetes.default\" external-dns String check external dns resolve from pod \"\" external-address String check ping connection to an external address \"114.114.114.114\" network-mode String The cni plugin current cluster used \"kube-ovn\" enable-metrics Bool Whether to support metrics query true ovs.timeout Int Timeout on JSON-RPC requests to OVS. 2 system.run.dir String OVS default run directory. \"/var/run/openvswitch\" database.vswitch.name String The name of OVS db. \"Open_vSwitch\" database.vswitch.socket.remote String JSON-RPC unix socket to OVS db. \"unix:/var/run/openvswitch/db.sock\" database.vswitch.file.data.path String OVS db file. \"/etc/openvswitch/conf.db\" database.vswitch.file.log.path String OVS db log file. \"/var/log/openvswitch/ovsdb-server.log\" database.vswitch.file.pid.path String OVS db process id file. \"/var/run/openvswitch/ovsdb-server.pid\" database.vswitch.file.system.id.path String OVS system id file. \"/etc/openvswitch/system-id.conf\" service.vswitchd.file.log.path String OVS vswitchd daemon log file. \"/var/log/openvswitch/ovs-vswitchd.log\" service.vswitchd.file.pid.path String OVS vswitchd daemon process id file. \"/var/run/openvswitch/ovs-vswitchd.pid\" service.ovncontroller.file.log.path String OVN controller daemon log file. \"/var/log/ovn/ovn-controller.log\" service.ovncontroller.file.pid.path String OVN controller daemon process id file. \"/var/run/ovn/ovn-controller.pid\" <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/reference/metrics/","title":"Metrics","text":"<p>This document lists all the monitoring metrics provided by Kube-OVN.</p>"},{"location":"en/reference/metrics/#ovn-monitor","title":"ovn-monitor","text":"<p>OVN status metrics:</p> Type Metric Description Gauge kube_ovn_ovn_status OVN Health Status. The values are: (2) for standby or follower, (1) for active or leader, (0) for unhealthy. Gauge kube_ovn_failed_req_count The number of failed requests to OVN stack. Gauge kube_ovn_log_file_size The size of a log file associated with an OVN component. Gauge kube_ovn_db_file_size The size of a database file associated with an OVN component. Gauge kube_ovn_chassis_info Whether the OVN chassis is up (1) or down (0), together with additional information about the chassis. Gauge kube_ovn_db_status The status of OVN NB/SB DB, (1) for healthy, (0) for unhealthy. Gauge kube_ovn_logical_switch_info The information about OVN logical switch. This metric is always up (1). Gauge kube_ovn_logical_switch_external_id Provides the external IDs and values associated with OVN logical switches. This metric is always up (1). Gauge kube_ovn_logical_switch_port_binding Provides the association between a logical switch and a logical switch port. This metric is always up (1). Gauge kube_ovn_logical_switch_tunnel_key The value of the tunnel key associated with the logical switch. Gauge kube_ovn_logical_switch_ports_num The number of logical switch ports connected to the OVN logical switch. Gauge kube_ovn_logical_switch_port_info The information about OVN logical switch port. This metric is always up (1). Gauge kube_ovn_logical_switch_port_tunnel_key The value of the tunnel key associated with the logical switch port. Gauge kube_ovn_cluster_enabled Is OVN clustering enabled (1) or not (0). Gauge kube_ovn_cluster_role A metric with a constant '1' value labeled by server role. Gauge kube_ovn_cluster_status A metric with a constant '1' value labeled by server status. Gauge kube_ovn_cluster_term The current raft term known by this server. Gauge kube_ovn_cluster_leader_self Is this server consider itself a leader (1) or not (0). Gauge kube_ovn_cluster_vote_self Is this server voted itself as a leader (1) or not (0). Gauge kube_ovn_cluster_election_timer The current election timer value. Gauge kube_ovn_cluster_log_not_committed The number of log entries not yet committed by this server. Gauge kube_ovn_cluster_log_not_applied The number of log entries not yet applied by this server. Gauge kube_ovn_cluster_log_index_start The log entry index start value associated with this server. Gauge kube_ovn_cluster_log_index_next The log entry index next value associated with this server. Gauge kube_ovn_cluster_inbound_connections_total The total number of inbound connections to the server. Gauge kube_ovn_cluster_outbound_connections_total The total number of outbound connections from the server. Gauge kube_ovn_cluster_inbound_connections_error_total The total number of failed inbound connections to the server. Gauge kube_ovn_cluster_outbound_connections_error_total The total number of failed outbound connections from the server."},{"location":"en/reference/metrics/#ovs-monitor","title":"ovs-monitor","text":"<p><code>ovsdb</code> and <code>vswitchd</code> status metrics:</p> Type Metric Description Gauge ovs_status OVS Health Status. The values are: health(1), unhealthy(0). Gauge ovs_info This metric provides basic information about OVS. It is always set to 1. Gauge failed_req_count The number of failed requests to OVS stack. Gauge log_file_size The size of a log file associated with an OVS component. Gauge db_file_size The size of a database file associated with an OVS component. Gauge datapath Represents an existing datapath. This metrics is always 1. Gauge dp_total Represents total number of datapaths on the system. Gauge dp_if Represents an existing datapath interface. This metrics is always 1. Gauge dp_if_total Represents the number of ports connected to the datapath. Gauge dp_flows_total The number of flows in a datapath. Gauge dp_flows_lookup_hit The number of incoming packets in a datapath matching existing flows in the datapath. Gauge dp_flows_lookup_missed The number of incoming packets in a datapath not matching any existing flow in the datapath. Gauge dp_flows_lookup_lost The number of incoming packets in a datapath destined for userspace process but subsequently dropped before reaching userspace. Gauge dp_masks_hit The total number of masks visited for matching incoming packets. Gauge dp_masks_total The number of masks in a datapath. Gauge dp_masks_hit_ratio The average number of masks visited per packet. It is the ration between hit and total number of packets processed by a datapath. Gauge interface Represents OVS interface. This is the primary metric for all other interface metrics. This metrics is always 1. Gauge interface_admin_state The administrative state of the physical network link of OVS interface. The values are: down(0), up(1), other(2). Gauge interface_link_state The state of the physical network link of OVS interface. The values are: down(0), up(1), other(2). Gauge interface_mac_in_use The MAC address in use by OVS interface. Gauge interface_mtu The currently configured MTU for OVS interface. Gauge interface_of_port Represents the OpenFlow port ID associated with OVS interface. Gauge interface_if_index Represents the interface index associated with OVS interface. Gauge interface_tx_packets Represents the number of transmitted packets by OVS interface. Gauge interface_tx_bytes Represents the number of transmitted bytes by OVS interface. Gauge interface_rx_packets Represents the number of received packets by OVS interface. Gauge interface_rx_bytes Represents the number of received bytes by OVS interface. Gauge interface_rx_crc_err Represents the number of CRC errors for the packets received by OVS interface. Gauge interface_rx_dropped Represents the number of input packets dropped by OVS interface. Gauge interface_rx_errors Represents the total number of packets with errors received by OVS interface. Gauge interface_rx_frame_err Represents the number of frame alignment errors on the packets received by OVS interface. Gauge interface_rx_missed_err Represents the number of packets with RX missed received by OVS interface. Gauge interface_rx_over_err Represents the number of packets with RX overrun received by OVS interface. Gauge interface_tx_dropped Represents the number of output packets dropped by OVS interface. Gauge interface_tx_errors Represents the total number of transmit errors by OVS interface. Gauge interface_collisions Represents the number of collisions on OVS interface."},{"location":"en/reference/metrics/#kube-ovn-pinger","title":"kube-ovn-pinger","text":"<p>Network quality related metrics:</p> Type Metric Description Gauge pinger_ovs_up If the ovs on the node is up Gauge pinger_ovs_down If the ovs on the node is down Gauge pinger_ovn_controller_up If the ovn_controller on the node is up Gauge pinger_ovn_controller_down If the ovn_controller on the node is down Gauge pinger_inconsistent_port_binding The number of mismatch port bindings between ovs and ovn-sb Gauge pinger_apiserver_healthy If the apiserver request is healthy on this node Gauge pinger_apiserver_unhealthy If the apiserver request is unhealthy on this node Histogram pinger_apiserver_latency_ms The latency ms histogram the node request apiserver Gauge pinger_internal_dns_healthy If the internal dns request is unhealthy on this node Gauge pinger_internal_dns_unhealthy If the internal dns request is unhealthy on this node Histogram pinger_internal_dns_latency_ms The latency ms histogram the node request internal dns Gauge pinger_external_dns_health If the external dns request is healthy on this node Gauge pinger_external_dns_unhealthy If the external dns request is unhealthy on this node Histogram pinger_external_dns_latency_ms The latency ms histogram the node request external dns Histogram pinger_pod_ping_latency_ms The latency ms histogram for pod peer ping Gauge pinger_pod_ping_lost_total The lost count for pod peer ping Gauge pinger_pod_ping_count_total The total count for pod peer ping Histogram pinger_node_ping_latency_ms The latency ms histogram for pod ping node Gauge pinger_node_ping_lost_total The lost count for pod ping node Gauge pinger_node_ping_count_total The total count for pod ping node Histogram pinger_external_ping_latency_ms The latency ms histogram for pod ping external address Gauge pinger_external_lost_total The lost count for pod ping external address"},{"location":"en/reference/metrics/#kube-ovn-controller","title":"kube-ovn-controller","text":"<p><code>kube-ovn-controller</code> status metrics\uff1a</p> Type Metric Description Histogram rest_client_request_latency_seconds Request latency in seconds. Broken down by verb and URL Counter rest_client_requests_total Number of HTTP requests, partitioned by status code, method, and host Counter lists_total Total number of API lists done by the reflectors Summary list_duration_seconds How long an API list takes to return and decode for the reflectors Summary items_per_list How many items an API list returns to the reflectors Counter watches_total Total number of API watches done by the reflectors Counter short_watches_total Total number of short API watches done by the reflectors Summary watch_duration_seconds How long an API watch takes to return and decode for the reflectors Summary items_per_watch How many items an API watch returns to the reflectors Gauge last_resource_version Last resource version seen for the reflectors Histogram ovs_client_request_latency_milliseconds The latency histogram for ovs request Gauge subnet_available_ip_count The available num of ip address in subnet Gauge subnet_used_ip_count The used num of ip address in subnet"},{"location":"en/reference/metrics/#kube-ovn-cni","title":"kube-ovn-cni","text":"<p><code>kube-ovn-cni</code> status metrics:</p> Type Metric Description Histogram cni_op_latency_seconds The latency seconds for cni operations Counter cni_wait_address_seconds_total Latency that cni wait controller to assign an address Counter cni_wait_connectivity_seconds_total Latency that cni wait address ready in overlay network Counter cni_wait_route_seconds_total Latency that cni wait controller to add routed annotation to pod Histogram rest_client_request_latency_seconds Request latency in seconds. Broken down by verb and URL Counter rest_client_requests_total Number of HTTP requests, partitioned by status code, method, and host Counter lists_total Total number of API lists done by the reflectors Summary list_duration_seconds How long an API list takes to return and decode for the reflectors Summary items_per_list How many items an API list returns to the reflectors Counter watches_total Total number of API watches done by the reflectors Counter short_watches_total Total number of short API watches done by the reflectors Summary watch_duration_seconds How long an API watch takes to return and decode for the reflectors Summary items_per_watch How many items an API watch returns to the reflectors Gauge last_resource_version Last resource version seen for the reflectors Histogram ovs_client_request_latency_milliseconds The latency histogram for ovs request <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/reference/ovs-ovn-customized/","title":"OVS/OVN Customization","text":"<p>Upstream OVN/OVS was originally designed with the goal of a general purpose SDN controller and data plane. Due to some specific usage of the Kubernetes network,Kube-OVN only focused on part of the features. In order to achieve better performance, stability and specific features, Kube-OVN has made some modifications to the upstream OVN/OVS. Users using their own OVN/OVS with Kube-OVN controllers need to be aware of the possible impact of the following changes:</p> <p>Did not merge into the upstream modification.</p> <ul> <li>38df6fa3f7 Adjust the election timer to avoid large-scale cluster election jitter.</li> <li>d4888c4e75 add fdb update logging.</li> <li>d4888c4e75 fdb: fix mac learning in environments with hairpin enabled.</li> <li>9a81b91368 ovsdb-tool: add optional server id parameter for \"join-cluster\" command.</li> <li>0700cb90f9 Destination non-service traffic bypasses conntrack to improve performance on a particular data path.</li> <li>c48049a64f ECMP algorithm is adjusted from <code>dp_hash</code> to <code>hash</code> to avoid the hash error problem in some kernels.</li> <li>64383c14a9 Fix kernel Crash issue under Windows.</li> <li>08a95db2ca Support for github action builds on Windows.</li> <li>680e77a190 Windows uses tcp listening by default.</li> <li>05e57b3227 add support for windows.</li> <li>0181b68be1 br-int controller: listen on 127.0.0.1:6653 by default.</li> <li>b3801ecb73 modify src route priority.</li> <li>977e569539 fix reaching resubmit limit in underlay.</li> <li>45a4a22161 ovn-nbctl: do not remove LB if vips is empty.</li> <li>540592b9ff Replaces the Mac address as the destination address after DNAT to reduce additional performance overhead.</li> <li>10972d9632 Fix vswitchd ofport_usage memory leak.</li> </ul> <p>Merged into upstream modification:</p> <ul> <li>20626ea909 Multicast traffic bypasses LB and ACL processing stages to improve specific data path performance.</li> <li>a2d9ff3ccd Deb build adds compile optimization options.</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/reference/tunnel-protocol/","title":"Tunnel Protocol Selection","text":"<p>Kube-OVN uses OVN/OVS as the data plane implementation and currently supports <code>Geneve</code>, <code>Vxlan</code> and <code>STT</code> tunnel encapsulation protocols. These three protocols differ in terms of functionality, performance and ease of use. This document will describe the differences in the use of the three protocols so that users can choose according to their situation.</p>"},{"location":"en/reference/tunnel-protocol/#geneve","title":"Geneve","text":"<p>The <code>Geneve</code> protocol is the default tunneling protocol selected during Kube-OVN deployment and is also the default recommended tunneling protocol for OVN. This protocol is widely supported in the kernel and can be accelerated using the generic offload capability of modern NICs. Since <code>Geneve</code> has a variable header, it is possible to use 24bit space to mark different datapaths users can create a larger number of virtual networks.</p> <p>If you are using Mellanox or Corigine SmartNIC OVS offload, <code>Geneve</code> requires a higher kernel version. Upstream kernel of 5.4 or higher, or other compatible kernels that backports this feature.</p> <p>Due to the use of UDP encapsulation, this protocol does not make good use of the TCP-related offloads of modern NICs when handling TCP over UDP, and consumes more CPU resources when handling large packets.</p>"},{"location":"en/reference/tunnel-protocol/#vxlan","title":"Vxlan","text":"<p><code>Vxlan</code> is a recently supported protocol in the upstream OVN, which is widely supported in the kernel and can be accelerated using the common offload capabilities of modern NICs. Due to the limited length of the protocol header and the additional space required for OVN orchestration, there is a limit to the number of datapaths that can be created, with a maximum of 4096 datapaths and a maximum of 4096 ports under each datapath. Also, <code>inport</code>-based ACLs are not supported due to header length limitations.</p> <p><code>Vxlan</code> offloading is supported in common kernels if using Mellanox or Corigine SmartNIC.</p> <p>Due to the use of UDP encapsulation, this protocol does not make good use of the TCP-related offloads of modern NICs when handling TCP over UDP, and consumes more CPU resources when handling large packets.</p>"},{"location":"en/reference/tunnel-protocol/#stt","title":"STT","text":"<p>The <code>STT</code> protocol is an early tunneling protocol supported by the OVN that uses TCP-like headers to take advantage of the TCP offload capabilities common to modern NICs and significantly increase TCP throughput. The protocol also has a long header to support full OVN capabilities and large-scale datapaths.</p> <p>This protocol is not supported in the kernel. To use it, you need to compile an additional OVS kernel module and recompile the new version of the kernel module when upgrading the kernel.</p> <p>This protocol is not currently supported by the SmartNic and cannot use the offloading capability of OVS offloading.</p>"},{"location":"en/reference/tunnel-protocol/#references","title":"References","text":"<ul> <li>https://ipwithease.com/vxlan-vs-geneve-understand-the-difference/</li> <li>OVN FAQ</li> <li>What is Geneve</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/reference/underlay-topology/","title":"Underlay Traffic Topology","text":"<p>This document describes the forwarding path of traffic in Underlay mode under different scenarios.</p>"},{"location":"en/reference/underlay-topology/#pods-in-same-node-and-same-subnet","title":"Pods in Same Node and Same Subnet","text":"<p>Internal logical switches exchange packets directly, without access to the external network.</p> <p></p>"},{"location":"en/reference/underlay-topology/#pods-in-different-nodes-and-same-subnet","title":"Pods in Different Nodes and Same Subnet","text":"<p>Packets enter the physic switch via the node NIC and are exchanged by the physic switch.</p> <p></p>"},{"location":"en/reference/underlay-topology/#pods-in-same-node-and-different-subnets","title":"Pods in Same Node and Different Subnets","text":"<p>Packets enter the physic network via the node NIC and are exchanged and routed and forwarded by physic switches and routers.</p> <p></p> <p>Here br-provider-1 and br-provider-2 can be the same OVS bridge\uff0cmultiple subnet can share a Provider Network\u3002</p>"},{"location":"en/reference/underlay-topology/#pods-in-different-nodes-and-different-subnets","title":"Pods in Different Nodes and Different Subnets","text":"<p>Packets enter the physic network via the node NIC and are exchanged and routed and forwarded by physic switches and routers.</p> <p></p>"},{"location":"en/reference/underlay-topology/#access-to-external","title":"Access to External","text":"<p>Packets enter the physic network via the node NIC and are exchanged and routed and forwarded by physic switches and routers.</p> <p></p> <p>The communication between nodes and Pods follows the same logic.</p>"},{"location":"en/reference/underlay-topology/#overview-without-vlan-tag","title":"Overview without Vlan Tag","text":""},{"location":"en/reference/underlay-topology/#overview-with-vlan-tag","title":"Overview with Vlan Tag","text":""},{"location":"en/reference/underlay-topology/#pod-visit-service-ip","title":"Pod visit Service IP","text":"<p>Kube-OVN configures load balancing for each Kubernetes Service on a logical switch on each subnet. When a Pod accesses other Pods by accessing the Service IP, a network packet is constructed with the Service IP as the destination address and the MAC address of the gateway as the destination MAC address. After the network packet enters the logical switch, load balancing will intercept and DNAT the network packet to modify the destination IP and port to the IP and port of one of the Endpoint corresponding to the Service. Since the logical switch does not modify the Layer 2 destination MAC address of the network packet, the network packet will still be delivered to the physic gateway after entering the physic switch, and the physic gateway will be required to forward the network packet.</p>"},{"location":"en/reference/underlay-topology/#service-backend-is-the-same-node-and-same-subnet-pod","title":"Service Backend is the Same Node and Same Subnet Pod","text":""},{"location":"en/reference/underlay-topology/#service-backend-is-the-same-node-and-different-subnets-pod","title":"Service Backend is the Same Node and Different Subnets Pod","text":"<p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/start/one-step-install/","title":"One-Click Installation","text":"<p>Kube-OVN provides a one-click installation script to help you quickly install a highly available, production-ready Kube-OVN container network with Overlay networking by default.</p> <p>Helm Chart installation is supported since Kube-OVN v1.12.0, and the default deployment is Overlay networking.</p> <p>If you need Underlay/Vlan networking as the default container network\uff0cplease read Underlay Installation</p> <p>Before installation please read Prerequisites first to make sure the environment is ready.</p>"},{"location":"en/start/one-step-install/#script-installation","title":"Script Installation","text":""},{"location":"en/start/one-step-install/#download-the-installation-script","title":"Download the installation script","text":"<p>We recommend using the stable release version for production environments, please use the following command to download:</p> <pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/images/install.sh\n</code></pre> <p>If you are interested in the latest features of the master branch, please use the following command to download:</p> <pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/images/install.sh\n</code></pre>"},{"location":"en/start/one-step-install/#modify-configuration-options","title":"Modify Configuration Options","text":"<p>Open the script using the editor and change the following variables to the expected:</p> <pre><code>REGISTRY=\"kubeovn\"                     # Image Repo \nVERSION=\"v1.13.0\"                      # Image Tag\nPOD_CIDR=\"10.16.0.0/16\"                # Default subnet CIDR don't overlay with SVC/NODE/JOIN CIDR\nSVC_CIDR=\"10.96.0.0/12\"                # Be consistent with apiserver's service-cluster-ip-range\nJOIN_CIDR=\"100.64.0.0/16\"              # Pod/Host communication Subnet CIDR, don't overlay with SVC/NODE/POD CIDR\nLABEL=\"node-role.kubernetes.io/master\" # The node label to deploy OVN DB\nIFACE=\"\"                               # The name of the host NIC used by the container network, or if empty use the NIC that host Node IP in Kubernetes\nTUNNEL_TYPE=\"geneve\"                   # Tunnel protocol\uff0cavailable options: geneve, vxlan or stt. stt requires compilation of ovs kernel module\n</code></pre> <p>You can also use regular expression to math NIC names\uff0csuch as <code>IFACE=enp6s0f0,eth.*</code>.</p>"},{"location":"en/start/one-step-install/#run-the-script","title":"Run the Script","text":"<p><code>bash install.sh</code></p> <p>Wait Kube-OVN ready.</p>"},{"location":"en/start/one-step-install/#helm-chart-installation","title":"Helm Chart Installation","text":"<p>Since the installation of Kube-OVN requires setting some parameters, to install Kube-OVN using Helm, you need to follow the steps below.</p>"},{"location":"en/start/one-step-install/#view-the-node-ip-address","title":"View the node IP address","text":"<pre><code>$ kubectl get node -o wide\nNAME                     STATUS     ROLES           AGE   VERSION   INTERNAL-IP   EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION      CONTAINER-RUNTIME\nkube-ovn-control-plane   NotReady   control-plane   20h   v1.26.0   172.18.0.3    &lt;none&gt;        Ubuntu 22.04.1 LTS   5.10.104-linuxkit   containerd://1.6.9\nkube-ovn-worker          NotReady   &lt;none&gt;          20h   v1.26.0   172.18.0.2    &lt;none&gt;        Ubuntu 22.04.1 LTS   5.10.104-linuxkit   containerd://1.6.9\n</code></pre>"},{"location":"en/start/one-step-install/#remove-cluster-master-node-taint","title":"Remove cluster master node taint","text":"<pre><code>$ kubectl taint node kube-ovn-control-plane node-role.kubernetes.io/control-plane:NoSchedule-\nnode/kube-ovn-control-plane untainted\n</code></pre> <p>This step can be skipped if you are sure that you do not need to schedule the pod at the master node.</p>"},{"location":"en/start/one-step-install/#add-label-to-node","title":"Add label to node","text":"<pre><code>$ kubectl label node -lbeta.kubernetes.io/os=linux kubernetes.io/os=linux --overwrite\nnode/kube-ovn-control-plane not labeled\nnode/kube-ovn-worker not labeled\n\n$ kubectl label node -lnode-role.kubernetes.io/control-plane kube-ovn/role=master --overwrite\nnode/kube-ovn-control-plane labeled\n\n# The following labels are used for the installation of dpdk images and can be ignored in non-dpdk cases\n$ kubectl label node -lovn.kubernetes.io/ovs_dp_type!=userspace ovn.kubernetes.io/ovs_dp_type=kernel --overwrite\nnode/kube-ovn-control-plane labeled\nnode/kube-ovn-worker labeled\n</code></pre>"},{"location":"en/start/one-step-install/#add-helm-repo-information","title":"Add Helm Repo information","text":"<pre><code>$ helm repo add kubeovn https://kubeovn.github.io/kube-ovn/\n\"kubeovn\" has been added to your repositories\n\n$ helm repo list\nNAME            URL\nkubeovn         https://kubeovn.github.io/kube-ovn/\n\n$ helm search repo kubeovn\nNAME                CHART VERSION   APP VERSION DESCRIPTION\nkubeovn/kube-ovn    0.1.0           1.12.0      Helm chart for Kube-OVN\n</code></pre>"},{"location":"en/start/one-step-install/#run-helm-install-to-install-kube-ovn","title":"Run helm install to install Kube-OVN","text":"<p>The Node0IP, Node1IP, and Node2IP parameters are the IP addresses of the cluster master nodes, respectively. For other parameters, you can refer to the variable definitions in the values.yaml file.</p> <pre><code># Single master node environment install\n$ helm install kube-ovn kubeovn/kube-ovn --set MASTER_NODES=${Node0IP}\n\n# Using the node information above as an example, execute the install command\n$ helm install kube-ovn kubeovn/kube-ovn --set MASTER_NODES=172.18.0.3\nNAME: kube-ovn\nLAST DEPLOYED: Fri Mar 31 12:43:43 2023\nNAMESPACE: default\nSTATUS: deployed\nREVISION: 1\nTEST SUITE: None\n\n# Highly Available Cluster Installation\n$ helm install kube-ovn kubeovn/kube-ovn --set MASTER_NODES=${Node0IP},${Node1IP},${Node2IP}, --set replicaCount=3\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/start/prepare/","title":"Prerequisites","text":"<p>Kube-OVN is a CNI-compliant network system that depends on the Kubernetes environment and the corresponding kernel network module for its operation. Below are the operating system and software versions tested, the environment configuration and the ports that need to be opened.</p>"},{"location":"en/start/prepare/#software-version","title":"Software Version","text":"<ul> <li>Kubernetes &gt;= 1.23.</li> <li>Docker &gt;= 1.12.6, Containerd &gt;= 1.3.4.</li> <li>OS: CentOS 7/8, Ubuntu 16.04/18.04/20.04.</li> <li>For other Linux distributions, please make sure <code>geneve</code>, <code>openvswitch</code>, <code>ip_tables</code> and <code>iptable_nat</code> kernel modules exist.</li> </ul> <p>Attention\uff1a</p> <ol> <li>For CentOS kernel version 3.10.0-862 bug exists in <code>netfilter</code> modules that lead Kube-OVN embed nat and lb failure.Please update kernel and check Floating IPs broken after kernel upgrade to Centos/RHEL 7.5 - DNAT not working.</li> <li>Kernel version 4.18.0-372.9.1.el8.x86_64 in Rocky Linux 8.6 has a TCP connection problem TCP connection failed in Rocky Linux 8.6\uff0cplease update kernel to 4.18.0-372.13.1.el8_6.x86_64 or later\u3002</li> <li>For kernel version 4.4, the related <code>openvswitch</code> module has some issues for ct\uff0cplease update kernel version or manually compile <code>openvswitch</code> kernel module.</li> <li>When building Geneve tunnel IPv6 in kernel should be enabled\uff0ccheck the kernel bootstrap options with <code>cat /proc/cmdline</code>.Check Geneve tunnels don't work when ipv6 is disabled for the detail bug info.</li> </ol>"},{"location":"en/start/prepare/#environment-setup","title":"Environment Setup","text":"<ul> <li>Kernel should enable IPv6, if kernel bootstrap options contain <code>ipv6.disable=1</code>, it should be set to <code>0</code>.</li> <li><code>kube-proxy</code> works, Kube-OVN can visit <code>kube-apiserver</code> from Service ClusterIP.</li> <li>Make sure kubelet enabled <code>CNI</code> and find cni-bin and cni-conf in default directories, kubelet bootstrap options should contain <code>--network-plugin=cni --cni-bin-dir=/opt/cni/bin --cni-conf-dir=/etc/cni/net.d</code>.</li> <li>Make sure no other CNI installed or has been removed\uff0ccheck if any config files still exist in<code>/etc/cni/net.d/</code>.</li> </ul>"},{"location":"en/start/prepare/#ports-need-open","title":"Ports Need Open","text":"Component Port Usage ovn-central 6641/tcp, 6642/tcp, 6643/tcp, 6644/tcp ovn-db and raft server listen ports ovs-ovn Geneve 6081/udp, STT 7471/tcp, Vxlan 4789/udp tunnel ports kube-ovn-controller 10660/tcp metrics port kube-ovn-daemon 10665/tcp metrics port kube-ovn-monitor 10661/tcp metrics port <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/start/sealos-install/","title":"One-Click Deployment of Kubernetes and Kube-OVN with sealos","text":"<p>sealos, a distribution of Kubernetes, helps users quickly initialize a container cluster from scratch. By using sealos, users can deploy a Kubernetes cluster with Kube-OVN installed in minutes with a single command.</p>"},{"location":"en/start/sealos-install/#download-sealos","title":"Download sealos","text":"AMD64 ARM64 <pre><code>wget https://github.com/labring/sealos/releases/download/v4.0.0/sealos_4.0.0_linux_amd64.tar.gz \\\n&amp;&amp; tar zxvf sealos_4.0.0_linux_amd64.tar.gz sealos &amp;&amp; chmod +x sealos &amp;&amp; mv sealos /usr/bin\n</code></pre> <pre><code>wget https://github.com/labring/sealos/releases/download/v4.0.0/sealos_4.0.0_linux_arm64.tar.gz \\\n&amp;&amp; tar zxvf sealos_4.0.0_linux_arm64.tar.gz sealos &amp;&amp; chmod +x sealos &amp;&amp; mv sealos /usr/bin\n</code></pre>"},{"location":"en/start/sealos-install/#deploy-kubernetes-and-kube-ovn","title":"Deploy Kubernetes and Kube-OVN","text":"<pre><code>```bash\nsealos run labring/kubernetes:v1.24.3 labring/kube-ovn:v1.10.5 \\\n  --masters [masters ips seperated by comma] \\\n  --nodes [nodes ips seperated by comma] -p [your-ssh-passwd]\n```\n</code></pre>"},{"location":"en/start/sealos-install/#wait-to-finish","title":"Wait to finish","text":"<pre><code>```bash\n[Step 6/6] Finish\n\n                    ,,,,\n                    ,::,\n                   ,,::,,,,\n            ,,,,,::::::::::::,,,,,\n         ,,,::::::::::::::::::::::,,,\n       ,,::::::::::::::::::::::::::::,,\n     ,,::::::::::::::::::::::::::::::::,,\n    ,::::::::::::::::::::::::::::::::::::,\n   ,:::::::::::::,,   ,,:::::,,,::::::::::,\n ,,:::::::::::::,       ,::,     ,:::::::::,\n ,:::::::::::::,   :x,  ,::  :,   ,:::::::::,\n,:::::::::::::::,  ,,,  ,::, ,,  ,::::::::::,\n,:::::::::::::::::,,,,,,:::::,,,,::::::::::::,    ,:,   ,:,            ,xx,                            ,:::::,   ,:,     ,:: :::,    ,x\n,::::::::::::::::::::::::::::::::::::::::::::,    :x: ,:xx:        ,   :xx,                          :xxxxxxxxx, :xx,   ,xx:,xxxx,   :x\n,::::::::::::::::::::::::::::::::::::::::::::,    :xxxxx:,  ,xx,  :x:  :xxx:x::,  ::xxxx:           :xx:,  ,:xxx  :xx, ,xx: ,xxxxx:, :x\n,::::::::::::::::::::::::::::::::::::::::::::,    :xxxxx,   :xx,  :x:  :xxx,,:xx,:xx:,:xx, ,,,,,,,,,xxx,    ,xx:   :xx:xx:  ,xxx,:xx::x\n,::::::,,::::::::,,::::::::,,:::::::,,,::::::,    :x:,xxx:  ,xx,  :xx  :xx:  ,xx,xxxxxx:, ,xxxxxxx:,xxx:,  ,xxx,    :xxx:   ,xxx, :xxxx\n,::::,    ,::::,   ,:::::,   ,,::::,    ,::::,    :x:  ,:xx,,:xx::xxxx,,xxx::xx: :xx::::x: ,,,,,,   ,xxxxxxxxx,     ,xx:    ,xxx,  :xxx\n,::::,    ,::::,    ,::::,    ,::::,    ,::::,    ,:,    ,:,  ,,::,,:,  ,::::,,   ,:::::,            ,,:::::,        ,,      :x:    ,::\n,::::,    ,::::,    ,::::,    ,::::,    ,::::,\n ,,,,,    ,::::,    ,::::,    ,::::,    ,:::,             ,,,,,,,,,,,,,\n          ,::::,    ,::::,    ,::::,    ,:::,        ,,,:::::::::::::::,\n          ,::::,    ,::::,    ,::::,    ,::::,  ,,,,:::::::::,,,,,,,:::,\n          ,::::,    ,::::,    ,::::,     ,::::::::::::,,,,,\n           ,,,,     ,::::,     ,,,,       ,,,::::,,,,\n                    ,::::,\n                    ,,::,\n\nThanks for choosing Kube-OVN!\nFor more advanced features, please read https://github.com/kubeovn/kube-ovn#documents\nIf you have any question, please file an issue https://github.com/kubeovn/kube-ovn/issues/new/choose\n2022-08-10T16:31:34 info succeeded in creating a new cluster, enjoy it!\n2022-08-10T16:31:34 info\n      ___           ___           ___           ___       ___           ___\n     /\\  \\         /\\  \\         /\\  \\         /\\__\\     /\\  \\         /\\  \\\n    /::\\  \\       /::\\  \\       /::\\  \\       /:/  /    /::\\  \\       /::\\  \\\n   /:/\\ \\  \\     /:/\\:\\  \\     /:/\\:\\  \\     /:/  /    /:/\\:\\  \\     /:/\\ \\  \\\n  _\\:\\~\\ \\  \\   /::\\~\\:\\  \\   /::\\~\\:\\  \\   /:/  /    /:/  \\:\\  \\   _\\:\\~\\ \\  \\\n /\\ \\:\\ \\ \\__\\ /:/\\:\\ \\:\\__\\ /:/\\:\\ \\:\\__\\ /:/__/    /:/__/ \\:\\__\\ /\\ \\:\\ \\ \\__\\\n \\:\\ \\:\\ \\/__/ \\:\\~\\:\\ \\/__/ \\/__\\:\\/:/  / \\:\\  \\    \\:\\  \\ /:/  / \\:\\ \\:\\ \\/__/\n  \\:\\ \\:\\__\\    \\:\\ \\:\\__\\        \\::/  /   \\:\\  \\    \\:\\  /:/  /   \\:\\ \\:\\__\\\n   \\:\\/:/  /     \\:\\ \\/__/        /:/  /     \\:\\  \\    \\:\\/:/  /     \\:\\/:/  /\n    \\::/  /       \\:\\__\\         /:/  /       \\:\\__\\    \\::/  /       \\::/  /\n     \\/__/         \\/__/         \\/__/         \\/__/     \\/__/         \\/__/\n\n                  Website :https://www.sealos.io/\n                  Address :github.com/labring/sealos\n```\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/start/underlay/","title":"Underlay Installation","text":"<p>By default, the default subnet uses Geneve to encapsulate cross-host traffic, and build an overlay network on top of the infrastructure.</p> <p>For the case that you want the container network to use the physical network address directly, you can set the default subnet of Kube-OVN to work in Underlay mode, which can directly assign the address resources in the physical network to the containers, achieving better performance and connectivity with the physical network.</p> <p></p>"},{"location":"en/start/underlay/#limitation","title":"Limitation","text":"<p>Since the container network in this mode uses physical network directly for L2 packet forwarding, L3 functions such as SNAT/EIP, distributed gateway/centralized gateway in Overlay mode cannot be used. VPC level isolation is also not available for underlay subnet.</p>"},{"location":"en/start/underlay/#comparison-with-macvlan","title":"Comparison with Macvlan","text":"<p>The Underlay mode of Kube-OVN is very similar to the Macvlan, with the following major differences in functionality and performance:</p> <ol> <li>Macvlan performs better in terms of throughput and latency performance metrics due to its shorter kernel path and the fact that it does not require OVS for packet processing.</li> <li>Kube-OVN provides arp-proxy functionality through flow tables to mitigate the risk of arp broadcast storms on large-scale networks.</li> <li>Since Macvlan works at the bottom of the kernel and bypasses the host netfilter, Service and NetworkPolicy functionality requires additional development.    Kube-OVN provides Service and NetworkPolicy capabilities through the OVS flow table.</li> <li>Kube-OVN Underlay mode provides additional features such as address management, fixed IP and QoS compared to Macvlan.</li> </ol>"},{"location":"en/start/underlay/#environment-requirements","title":"Environment Requirements","text":"<p>In Underlay mode, the OVS will bridge a node NIC to the OVS bridge and send packets directly through that node NIC, relying on the underlying network devices for L2/L3 level forwarding capabilities. You need to configure the corresponding gateway, Vlan and security policy in the underlying network device in advance.</p> <ol> <li>For OpenStack VM environments, you need to turn off <code>PortSecurity</code> on the corresponding network port.</li> <li>For VMware vSwitch networks, <code>MAC Address Changes</code>, <code>Forged Transmits</code> and <code>Promiscuous Mode Operation</code> should be set to <code>allow</code>.</li> <li>For Hyper-V virtualization,  <code>MAC Address Spoofing</code> should be enabled in VM nic advanced features.</li> <li>Public clouds, such as AWS, GCE, AliCloud, etc., do not support user-defined Mac, so they cannot support Underlay mode network. In this scenario, if you want to use Underlay, it is recommended to use the VPC-CNI provided by the corresponding public cloud vendor..</li> <li>The network interface that is bridged into ovs can not be type of Linux Bridge.</li> </ol> <p>For management and container networks using the same NIC, Kube-OVN will transfer the NIC's Mac address, IP address, route, and MTU to the corresponding OVS Bridge to support single NIC deployment of Underlay networks. OVS Bridge name format is <code>br-PROVIDER_NAME</code>\uff0c<code>PROVIDER_NAME</code> is the name of <code>ProviderNetwork</code> (Default: provider).</p>"},{"location":"en/start/underlay/#specify-network-mode-when-deploying","title":"Specify Network Mode When Deploying","text":"<p>This deployment mode sets the default subnet to Underlay mode, and all Pods with no subnet specified will run in the Underlay network by default.</p>"},{"location":"en/start/underlay/#download-script","title":"Download Script","text":"<pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/images/install.sh\n</code></pre>"},{"location":"en/start/underlay/#modify-configuration-options","title":"Modify Configuration Options","text":"<pre><code>ENABLE_ARP_DETECT_IP_CONFLICT # disable vlan arp conflict detection if necessary\nNETWORK_TYPE                  # set to vlan\nVLAN_INTERFACE_NAME           # set to the NIC that carries the Underlay traffic, e.g. eth1\nVLAN_ID                       # The VLAN Tag need to be added\uff0cif set 0 no vlan tag will be added\nPOD_CIDR                      # The Underlay network CIDR\uff0c e.g. 192.168.1.0/24\nPOD_GATEWAY                   # Underlay physic gateway address, e.g. 192.168.1.1\nEXCLUDE_IPS                   # Exclude ranges to avoid conflicts between container network and IPs already in use on the physical network, e.g. 192.168.1.1..192.168.1.100\nENABLE_LB                     # If Underlay Subnet needs to visit Service set it to true\nEXCHANGE_LINK_NAME            # If swap the names of the OVS bridge and the bridge interface under the default provider-network. Default to false.\nLS_DNAT_MOD_DL_DST            # If DNAT translate MAC addresses to accelerate service access. Default to true.\n</code></pre>"},{"location":"en/start/underlay/#run-the-script","title":"Run the Script","text":"<pre><code>bash install.sh\n</code></pre>"},{"location":"en/start/underlay/#dynamically-create-underlay-networks-via-crd","title":"Dynamically Create Underlay Networks via CRD","text":"<p>This approach dynamically creates an Underlay subnet that Pod can use after installation.</p>"},{"location":"en/start/underlay/#create-providernetwork","title":"Create ProviderNetwork","text":"<p>ProviderNetwork provides the abstraction of host NIC to physical network mapping, unifies the management of NICs belonging to the same network, and solves the configuration problems in complex environments with multiple NICs on the same machine, inconsistent NIC names and inconsistent corresponding Underlay networks.</p> <p>Create ProviderNetwork as below:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: ProviderNetwork\nmetadata:\n  name: net1\nspec:\n  defaultInterface: eth1\n  customInterfaces:\n    - interface: eth2\n      nodes:\n        - node1\n  excludeNodes:\n    - node2\n</code></pre> <p>Note: The length of the ProviderNetwork resource name must not exceed 12.</p> <ul> <li><code>defaultInterface</code>: The default node NIC name. When the ProviderNetwork is successfully created, an OVS bridge named br-net1 (in the format <code>br-NAME</code>) is created in each node (except excludeNodes) and the specified node NIC is bridged to this bridge.</li> <li><code>customInterfaces</code>: Optionally, you can specify the NIC to be used for a specific node.</li> <li><code>excludeNodes</code>: Optional, to specify nodes that do not bridge the NIC. Nodes in this list will be added with the <code>net1.provider-network.ovn.kubernetes.io/exclude=true</code> tag.</li> </ul> <p>Other nodes will be added with the following tags:</p> Key Value Description net1.provider-network.ovn.kubernetes.io/ready true bridge work finished, ProviderNetwork is ready on this node net1.provider-network.ovn.kubernetes.io/interface eth1 The name of the bridged NIC in the node. net1.provider-network.ovn.kubernetes.io/mtu 1500 MTU of bridged NIC in node <p>If an IP has been configured on the node NIC, the IP address and the route on the NIC are transferred to the corresponding OVS bridge.</p>"},{"location":"en/start/underlay/#create-vlan","title":"Create VLAN","text":"<p>Vlan provides an abstraction to bind Vlan Tag and ProviderNetwork.</p> <p>Create a VLAN as below:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Vlan\nmetadata:\n  name: vlan1\nspec:\n  id: 0\n  provider: net1\n</code></pre> <ul> <li><code>id</code>: VLAN ID/Tag\uff0cKube-OVN will add this Vlan tag to traffic, if set 0, no tag is added.</li> <li><code>provider</code>: The name of ProviderNetwork. Multiple VLAN can use a same ProviderNetwork.</li> </ul>"},{"location":"en/start/underlay/#create-subnet","title":"Create Subnet","text":"<p>Bind Vlan to a Subnet as below\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: subnet1\nspec:\nprotocol: IPv4\ncidrBlock: 172.17.0.0/16\ngateway: 172.17.0.1\nvlan: vlan1\n</code></pre> <p>Simply specify the value of <code>vlan</code> as the name of the VLAN to be used. Multiple subnets can refer to the same VLAN.</p>"},{"location":"en/start/underlay/#create-pod","title":"Create Pod","text":"<p>You can create containers in the normal way, check whether the container IP is in the specified range and whether the container can interoperate with the physical network.</p> <p>For fixed IP requirements, please refer to Fixed Addresses</p>"},{"location":"en/start/underlay/#logical-gateway","title":"Logical Gateway","text":"<p>For cases where no gateway exists in the physical network, Kube-OVN supports the use of logical gateways configured in the subnet in Underlay mode. To use this feature, set <code>spec.logicalGateway</code> to <code>true</code> for the subnet:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: subnet1\nspec:\nprotocol: IPv4\ncidrBlock: 172.17.0.0/16\ngateway: 172.17.0.1\nvlan: vlan1\nlogicalGateway: true\n</code></pre> <p>When this feature is turned on, the Pod does not use an external gateway, but a Logical Router created by Kube-OVN to forward cross-subnet communication.</p>"},{"location":"en/start/underlay/#interconnection-of-underlay-and-overlay-networks","title":"Interconnection of Underlay and Overlay Networks","text":"<p>If a cluster has both Underlay and Overlay subnets, by default, Pods in the Overlay subnet can access the Pod IPs in the Underlay subnet via a gateway using NAT. From the perspective of Pods in the Underlay subnet, the addresses in the Overlay subnet are external, and require the underlying physical device to forward, but the underlying physical device does not know the addresses in the Overlay subnet and cannot forward. Therefore, Pods in the Underlay subnet cannot access Pods in the Overlay subnet directly via Pod IPs.</p> <p>If you need to enable communication between Underlay and Overlay networks, you need to set the <code>u2oInterconnection</code> of the subnet to <code>true</code>. In this case, Kube-OVN will use an additional Underlay IP to connect the Underlay subnet and the <code>ovn-cluster</code> logical router, and set the corresponding routing rules to enable communication. Unlike the logical gateway, this solution only connects the Underlay and Overlay subnets within Kube-OVN, and other traffic accessing the Internet will still be forwarded through the physical gateway.</p>"},{"location":"en/start/underlay/#specify-logical-gateway-ip","title":"Specify logical gateway IP","text":"<p>After the interworking function is enabled, an IP from the subnet will be randomly selected as the logical gateway. If you need to specify the logical gateway of the Underlay Subnet, you can specify the field <code>u2oInterconnectionIP</code>.</p>"},{"location":"en/start/underlay/#specify-custom-vpc-for-underlay-subnet-connection","title":"Specify custom VPC for Underlay Subnet connection","text":"<p>By default, the Underlay Subnet will communicate with the Overlay Subnet on the default VPC. If you want to specify to communicate with a certain VPC, after setting <code>u2oInterconnection</code> to <code>true</code>, specify the <code>subnet.spec.vpc</code> field as the name of the VPC.</p>"},{"location":"en/start/underlay/#notice","title":"Notice","text":"<p>If you have an IP address configured on the network card of the node you are using, and the operating system configures the network using Netplan (such as Ubuntu), it is recommended that you set the renderer of Netplan to NetworkManager and configure a static IP address for the node's network card (disable DHCP).</p> <pre><code>network:\nrenderer: NetworkManager\nethernets:\neth0:\ndhcp4: no\naddresses:\n- 172.16.143.129/24\nversion: 2\n</code></pre> <p>If you want to modify the IP or routing configuration of the network card, you need to execute the following commands after modifying the Netplan configuration:</p> <pre><code>netplan generate\n\nnmcli connection reload netplan-eth0\nnmcli device set eth0 managed yes\n</code></pre> <p>After executing the above commands, Kube-OVN will transfer the IP and routing from the network card to the OVS bridge.</p> <p>If your operating system manages the network using NetworkManager (such as CentOS), you need to execute the following command after modifying the network card configuration:</p> <pre><code>nmcli connection reload eth0\nnmcli device set eth0 managed yes\nnmcli -t -f GENERAL.STATE device show eth0 | grep -qw unmanaged || nmcli device reapply eth0\n</code></pre> <p>Notice\uff1aIf the host nic's MAC is changed, Kube-OVN will not change the OVS bridge's MAC unless kube-ovn-cni is restarted.</p>"},{"location":"en/start/underlay/#known-issues","title":"Known Issues","text":""},{"location":"en/start/underlay/#when-the-physical-network-is-enabled-with-hairpin-pod-network-is-abnormal","title":"When the physical network is enabled with hairpin, Pod network is abnormal","text":"<p>When physical networks enable hairpin or similar behaviors, problems such as gateway check failure when creating Pods and abnormal network communication of Pods may occur. This is because the default MAC learning function of OVS bridge does not support this kind of network environment.</p> <p>To solve this problem, it is necessary to turn off hairpin (or modify the relevant configuration of physical network), or update the Kube-OVN version.</p>"},{"location":"en/start/underlay/#when-there-are-a-large-number-of-pods-gateway-check-for-new-pods-fails","title":"When there are a large number of Pods, gateway check for new Pods fails","text":"<p>If there are a large number of Pods running on the same node (more than 300), it may cause packet loss due to the OVS flow table resubmit times exceeding the upper limit of ARP broadcast packets.</p> <pre><code>2022-11-13T08:43:46.782Z|00222|ofproto_dpif_upcall(handler5)|WARN|Flow: arp,in_port=331,vlan_tci=0x0000,dl_src=00:00:00:25:eb:39,dl_dst=ff:ff:ff:ff:ff:ff,arp_spa=10.213.131.240,arp_tpa=10.213.159.254,arp_op=1,arp_sha=00:00:00:25:eb:39,arp_tha=ff:ff:ff:ff:ff:ff\n\nbridge(\"br-int\")\n----------------\n 0. No match.\n     &gt;&gt;&gt;&gt; received packet on unknown port 331 &lt;&lt;&lt;&lt;\n    drop\n\nFinal flow: unchanged\nMegaflow: recirc_id=0,eth,arp,in_port=331,dl_src=00:00:00:25:eb:39\nDatapath actions: drop\n2022-11-13T08:44:34.077Z|00224|ofproto_dpif_xlate(handler5)|WARN|over 4096 resubmit actions on bridge br-int while processing arp,in_port=13483,vlan_tci=0x0000,dl_src=00:00:00:59:ef:13,dl_dst=ff:ff:ff:ff:ff:ff,arp_spa=10.213.152.3,arp_tpa=10.213.159.254,arp_op=1,arp_sha=00:00:00:59:ef:13,arp_tha=ff:ff:ff:ff:ff:ff\n</code></pre> <p>To solve this issue, modify the OVN NB option <code>bcast_arp_req_flood</code> to <code>false</code>:</p> <pre><code>kubectl ko nbctl set NB_Global . options:bcast_arp_req_flood=false\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/start/uninstall/","title":"Uninstall","text":"<p>If you need to remove the Kube-OVN and replace it with another network plugin, please follow the steps below to remove all the corresponding Kube-OVN component and OVS configuration to avoid interference with other network plugins.</p> <p>Feel free to contact us with an Issue to give us feedback on why you don't use Kube-OVN to help us improve it.</p>"},{"location":"en/start/uninstall/#delete-resource-in-kubernetes","title":"Delete Resource in Kubernetes","text":"<p>Download and run the script below to delete resource created in Kubernetes:</p> <pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/images/cleanup.sh\nbash cleanup.sh\n</code></pre>"},{"location":"en/start/uninstall/#cleanup-config-and-logs-on-every-node","title":"Cleanup Config and Logs on Every Node","text":"<p>Run the following commands on each node to clean up the configuration retained by ovsdb and openvswitch:</p> <pre><code>rm -rf /var/run/openvswitch\nrm -rf /var/run/ovn\nrm -rf /etc/origin/openvswitch/\nrm -rf /etc/origin/ovn/\nrm -rf /etc/cni/net.d/00-kube-ovn.conflist\nrm -rf /etc/cni/net.d/01-kube-ovn.conflist\nrm -rf /var/log/openvswitch\nrm -rf /var/log/ovn\nrm -fr /var/log/kube-ovn\n</code></pre>"},{"location":"en/start/uninstall/#reboot-node","title":"Reboot Node","text":"<p>Reboot the machine to ensure that the corresponding NIC information and iptable/ipset rules are cleared to avoid the interference with other network plugins:</p> <pre><code>reboot\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"}]}
\ No newline at end of file
+{"config":{"lang":["ja","en"],"separator":"[\\s\\u200b\\-]","pipeline":["stemmer"]},"docs":[{"location":"","title":"Kube-OVN","text":"<p>Kube-OVN \u662f\u4e00\u6b3e CNCF \u65d7\u4e0b\u7684\u4f01\u4e1a\u7ea7\u4e91\u539f\u751f\u7f51\u7edc\u7f16\u6392\u7cfb\u7edf\uff0c\u5c06 SDN \u7684\u80fd\u529b\u548c\u4e91\u539f\u751f\u7ed3\u5408\uff0c \u63d0\u4f9b\u4e30\u5bcc\u7684\u529f\u80fd\uff0c\u6781\u81f4\u7684\u6027\u80fd\u4ee5\u53ca\u826f\u597d\u7684\u53ef\u8fd0\u7ef4\u6027\u3002</p> <p>\u4e30\u5bcc\u7684\u529f\u80fd\uff1a</p> <p>\u5982\u679c\u4f60\u6000\u5ff5 SDN \u9886\u57df\u4e30\u5bcc\u7684\u7f51\u7edc\u80fd\u529b\u5374\u5728\u4e91\u539f\u751f\u9886\u57df\u82e6\u82e6\u8ffd\u5bfb\u800c\u4e0d\u5f97\uff0c\u90a3\u4e48 Kube-OVN \u5c06\u662f\u4f60\u7684\u6700\u4f73\u9009\u62e9\u3002</p> <p>\u501f\u52a9 OVS/OVN \u5728 SDN \u9886\u57df\u6210\u719f\u7684\u80fd\u529b\uff0cKube-OVN \u5c06\u7f51\u7edc\u865a\u62df\u5316\u7684\u4e30\u5bcc\u529f\u80fd\u5e26\u5165\u4e91\u539f\u751f\u9886\u57df\u3002\u76ee\u524d\u5df2\u652f\u6301\u5b50\u7f51\u7ba1\u7406\uff0c \u9759\u6001 IP \u5206\u914d\uff0c\u5206\u5e03\u5f0f/\u96c6\u4e2d\u5f0f\u7f51\u5173\uff0cUnderlay/Overlay \u6df7\u5408\u7f51\u7edc\uff0c VPC \u591a\u79df\u6237\u7f51\u7edc\uff0c\u8de8\u96c6\u7fa4\u4e92\u8054\u7f51\u7edc\uff0cQoS \u7ba1\u7406\uff0c \u591a\u7f51\u5361\u7ba1\u7406\uff0cACL \u7f51\u7edc\u63a7\u5236\uff0c\u6d41\u91cf\u955c\u50cf\uff0cARM \u652f\u6301\uff0c Windows \u652f\u6301\u7b49\u8bf8\u591a\u529f\u80fd\u3002</p> <p>\u6781\u81f4\u7684\u6027\u80fd\uff1a</p> <p>\u5982\u679c\u4f60\u62c5\u5fc3\u5bb9\u5668\u7f51\u7edc\u4f1a\u5e26\u6765\u989d\u5916\u7684\u6027\u80fd\u635f\u8017\uff0c\u90a3\u4e48\u6765\u770b\u4e00\u4e0b Kube-OVN \u662f\u5982\u4f55\u6781\u81f4\u7684\u4f18\u5316\u6027\u80fd\u3002</p> <p>\u5728\u6570\u636e\u5e73\u9762\uff0c\u901a\u8fc7\u4e00\u7cfb\u5217\u5bf9\u6d41\u8868\u548c\u5185\u6838\u7684\u7cbe\u5fc3\u4f18\u5316\uff0c\u5e76\u501f\u52a9 eBPF\u3001DPDK\u3001\u667a\u80fd\u7f51\u5361\u5378\u8f7d\u7b49\u65b0\u5174\u6280\u672f\uff0c Kube-OVN \u53ef\u4ee5\u5728\u5ef6\u8fdf\u548c\u541e\u5410\u91cf\u7b49\u65b9\u9762\u7684\u6307\u6807\u8fbe\u5230\u8fd1\u4f3c\u6216\u8d85\u51fa\u5bbf\u4e3b\u673a\u7f51\u7edc\u6027\u80fd\u7684\u6c34\u5e73\u3002\u5728\u63a7\u5236\u5e73\u9762\uff0c\u901a\u8fc7\u5bf9 OVN \u4e0a\u6e38\u6d41\u8868\u7684\u88c1\u526a\uff0c \u5404\u79cd\u7f13\u5b58\u6280\u672f\u7684\u4f7f\u7528\u548c\u8c03\u4f18\uff0cKube-OVN \u53ef\u4ee5\u652f\u6301\u5927\u89c4\u6a21\u4e0a\u5343\u8282\u70b9\u548c\u4e0a\u4e07 Pod \u7684\u96c6\u7fa4\u3002</p> <p>\u6b64\u5916 Kube-OVN \u8fd8\u5728\u4e0d\u65ad\u4f18\u5316 CPU \u548c\u5185\u5b58\u7b49\u8d44\u6e90\u7684\u4f7f\u7528\u91cf\uff0c\u4ee5\u9002\u5e94\u8fb9\u7f18\u7b49\u8d44\u6e90\u6709\u9650\u573a\u666f\u3002</p> <p>\u826f\u597d\u7684\u53ef\u8fd0\u7ef4\u6027\uff1a</p> <p>\u5982\u679c\u4f60\u5bf9\u5bb9\u5668\u7f51\u7edc\u7684\u8fd0\u7ef4\u5fc3\u5b58\u5fe7\u8651\uff0cKube-OVN \u5185\u7f6e\u4e86\u5927\u91cf\u7684\u5de5\u5177\u6765\u5e2e\u52a9\u4f60\u7b80\u5316\u8fd0\u7ef4\u64cd\u4f5c\u3002</p> <p>Kube-OVN \u63d0\u4f9b\u4e86\u4e00\u952e\u5b89\u88c5\u811a\u672c\uff0c\u5e2e\u52a9\u7528\u6237\u8fc5\u901f\u642d\u5efa\u751f\u4ea7\u5c31\u7eea\u7684\u5bb9\u5668\u7f51\u7edc\u3002\u540c\u65f6\u5185\u7f6e\u7684\u4e30\u5bcc\u7684\u76d1\u63a7\u6307\u6807\u548c Grafana \u9762\u677f\uff0c \u53ef\u5e2e\u52a9\u7528\u6237\u5efa\u7acb\u5b8c\u5584\u7684\u76d1\u63a7\u4f53\u7cfb\u3002\u5f3a\u5927\u7684\u547d\u4ee4\u884c\u5de5\u5177\u53ef\u4ee5\u7b80\u5316\u7528\u6237\u7684\u65e5\u5e38\u8fd0\u7ef4\u64cd\u4f5c\u3002\u901a\u8fc7\u548c Cilium \u7ed3\u5408\uff0c\u5229\u7528 eBPF \u80fd\u529b\u7528\u6237\u53ef\u4ee5 \u589e\u5f3a\u5bf9\u7f51\u7edc\u7684\u53ef\u89c2\u6d4b\u6027\u3002 \u6b64\u5916\u6d41\u91cf\u955c\u50cf\u7684\u80fd\u529b\u53ef\u4ee5\u65b9\u4fbf\u7528\u6237\u81ea\u5b9a\u4e49\u6d41\u91cf\u76d1\u63a7\uff0c\u5e76\u548c\u4f20\u7edf\u7684 NPM \u7cfb\u7edf\u5bf9\u63a5\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"contact/","title":"\u8054\u7cfb\u65b9\u5f0f","text":"<p>\u5173\u6ce8\u516c\u4f17\u53f7\u83b7\u5f97\u66f4\u591a\u6700\u65b0\u4fe1\u606f\uff0c\u8bf7\u626b\u63cf\u4e0b\u65b9\u4e8c\u7ef4\u7801:</p> <p></p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/accelerate-intra-node-tcp-with-ebpf/","title":"\u4f7f\u7528 eBPF \u52a0\u901f\u8282\u70b9\u5185 TCP \u901a\u4fe1","text":"<p>\u5728\u4e00\u4e9b\u8fb9\u7f18\u548c 5G \u7684\u573a\u666f\u4e0b\uff0c\u540c\u8282\u70b9\u5185\u7684 Pod \u4e4b\u95f4\u4f1a\u8fdb\u884c\u5927\u91cf\u7684 TCP \u901a\u4fe1\uff0c\u901a\u8fc7\u4f7f\u7528 Intel \u5f00\u6e90\u7684 istio-tcpip-bypass \u9879\u76ee\uff0cPod \u53ef\u4ee5\u501f\u52a9 eBPF \u7684\u80fd\u529b\u7ed5\u8fc7\u4e3b\u673a\u7684 TCP/IP \u534f\u8bae\u6808\uff0c\u76f4\u63a5\u8fdb\u884c socket \u901a\u4fe1\uff0c\u4ece\u800c\u5927\u5e45\u964d\u4f4e\u5ef6\u8fdf\u5e76\u63d0\u5347\u541e\u5410\u91cf\u3002</p>"},{"location":"advance/accelerate-intra-node-tcp-with-ebpf/#_1","title":"\u57fa\u672c\u539f\u7406","text":"<p>\u5728\u5f53\u524d\u7684\u5b9e\u73b0\u4e0b\uff0c\u540c\u4e3b\u673a\u7684\u4e24\u4e2a Pod \u8fdb\u884c TCP \u8fdb\u884c\u901a\u4fe1\u9700\u8981\u7ecf\u8fc7\u5927\u91cf\u7684\u7f51\u7edc\u6808\uff0c\u5305\u62ec TCP/IP, netfilter\uff0cOVS \u7b49\u5982\u4e0b\u56fe\u6240\u793a\uff1a</p> <p></p> <p>istio-tcpip-bypass \u63d2\u4ef6\u53ef\u4ee5\u81ea\u52a8\u5206\u6790\u5e76\u8bc6\u522b\u51fa\u540c\u4e3b\u673a\u5185\u7684 TCP \u901a\u4fe1\uff0c\u5e76\u7ed5\u8fc7\u590d\u6742\u7684\u5185\u6838\u6808\u4ece\u800c\u53ef\u4ee5\u76f4\u63a5\u8fdb\u884c socket \u95f4\u7684\u6570\u636e\u4f20\u8f93\uff0c \u6765\u964d\u4f4e\u7f51\u7edc\u6808\u5904\u7406\u5f00\u9500\uff0c\u5982\u4e0b\u56fe\u6240\u793a\uff1a</p> <p></p> <p>\u7531\u4e8e\u8be5\u7ec4\u4ef6\u53ef\u4ee5\u81ea\u52a8\u8bc6\u522b\u540c\u4e3b\u673a\u5185\u7684 TCP \u901a\u4fe1\uff0c\u5e76\u8fdb\u884c\u4f18\u5316\u3002\u5728\u57fa\u4e8e\u4ee3\u7406\u6a21\u5f0f\u7684 Service Mesh \u73af\u5883\u4e0b\uff0c\u8be5\u7ec4\u4ef6\u4e5f\u53ef\u4ee5\u589e\u5f3a Service Mesh \u7684\u6027\u80fd\u8868\u73b0\u3002</p> <p>\u66f4\u591a\u6280\u672f\u5b9e\u73b0\u7ec6\u8282\u53ef\u4ee5\u53c2\u8003 Tanzu Service Mesh Acceleration using eBPF\u3002</p>"},{"location":"advance/accelerate-intra-node-tcp-with-ebpf/#_2","title":"\u73af\u5883\u51c6\u5907","text":"<p>eBPF \u5bf9\u5185\u6838\u7248\u672c\u6709\u4e00\u5b9a\u8981\u6c42\uff0c\u63a8\u8350\u4f7f\u7528 Ubuntu 20.04 \u548c Linux 5.4.0-74-generic \u7248\u672c\u5185\u6838\u8fdb\u884c\u5b9e\u9a8c\u3002</p>"},{"location":"advance/accelerate-intra-node-tcp-with-ebpf/#_3","title":"\u5b9e\u9a8c\u6b65\u9aa4","text":"<p>\u5728\u540c\u4e00\u4e2a\u8282\u70b9\u4e0a\u90e8\u7f72\u4e24\u4e2a\u6027\u80fd\u6d4b\u8bd5 Pod\uff0c\u82e5\u96c6\u7fa4\u5185\u5b58\u5728\u591a\u53f0\u673a\u5668\u9700\u8981\u6307\u5b9a <code>nodeSelector</code>\uff1a</p> <pre><code># kubectl create deployment perf --image=kubeovn/perf:dev --replicas=2\ndeployment.apps/perf created\n# kubectl get pod -o wide\nNAME                    READY   STATUS    RESTARTS   AGE   IP           NODE     NOMINATED NODE   READINESS GATES\nperf-7697bc6ddf-b2cpv   1/1     Running   0          28s   100.64.0.3   sealos   &lt;none&gt;           &lt;none&gt;\nperf-7697bc6ddf-p2xpt   1/1     Running   0          28s   100.64.0.2   sealos   &lt;none&gt;           &lt;none&gt;\n</code></pre> <p>\u8fdb\u5165\u5176\u4e2d\u4e00\u4e2a Pod \u5f00\u542f qperf server\uff0c\u5728\u53e6\u4e00\u4e2a Pod \u4e2d\u542f\u52a8 qperf client \u8fdb\u884c\u6027\u80fd\u6d4b\u8bd5\uff1a</p> <pre><code># kubectl exec -it perf-7697bc6ddf-b2cpv sh\n/ # qperf\n\n# kubectl exec -it perf-7697bc6ddf-p2xpt sh\n/ # qperf -t 60 100.64.0.3 -ub -oo msg_size:1:16K:*4 -vu tcp_lat tcp_bw\n</code></pre> <p>\u90e8\u7f72 istio-tcpip-bypass \u63d2\u4ef6\uff1a</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/intel/istio-tcpip-bypass/main/bypass-tcpip-daemonset.yaml\n</code></pre> <p>\u518d\u6b21\u8fdb\u5165 perf client \u5bb9\u5668\u8fdb\u884c\u6027\u80fd\u6d4b\u8bd5\uff1a</p> <pre><code># kubectl exec -it perf-7697bc6ddf-p2xpt sh\n/ # qperf -t 60 100.64.0.3 -ub -oo msg_size:1:16K:*4 -vu tcp_lat tcp_bw\n</code></pre>"},{"location":"advance/accelerate-intra-node-tcp-with-ebpf/#_4","title":"\u6d4b\u8bd5\u7ed3\u679c","text":"<p>\u6839\u636e\u6d4b\u8bd5\u7ed3\u679c TCP \u5ef6\u8fdf\u5728\u4e0d\u540c\u6570\u636e\u5305\u5927\u5c0f\u7684\u60c5\u51b5\u4e0b\u4f1a\u6709 40% ~ 60% \u7684\u5ef6\u8fdf\u4e0b\u964d\uff0c\u5728\u6570\u636e\u5305\u5927\u4e8e 1024 \u5b57\u8282\u65f6\u541e\u5410\u91cf\u4f1a\u6709 40% ~ 80% \u63d0\u5347\u3002</p> Packet Size (byte) eBPF tcp_lat (us) Default tcp_lat (us) eBPF tcp_bw (Mb/s) Default tcp_bw(Mb/s) 1 20.2 44.5 1.36 4.27 4 20.2 48.7 5.48 16.7 16 19.6 41.6 21.7 63.5 64 18.8 41.3 96.8 201 256 19.2 36 395 539 1024 18.3 42.4 1360 846 4096 16.5 62.6 4460 2430 16384 20.2 58.8 9600 6900 <p>\u5728\u6d4b\u8bd5\u7684\u786c\u4ef6\u73af\u5883\u4e0b\uff0c\u6570\u636e\u5305\u5c0f\u4e8e 512 \u5b57\u8282\u65f6\uff0c\u4f7f\u7528 eBPF \u4f18\u5316\u541e\u5410\u91cf\u6307\u6807\u4f1a\u4f4e\u4e8e\u9ed8\u8ba4\u914d\u7f6e\u4e0b\u7684\u541e\u5410\u91cf\u3002 \u8be5\u60c5\u51b5\u53ef\u80fd\u548c\u9ed8\u8ba4\u914d\u7f6e\u4e0b\u7f51\u5361\u5f00\u542f TCP \u805a\u5408\u4f18\u5316\u76f8\u5173\u3002\u5982\u679c\u5e94\u7528\u573a\u666f\u5bf9\u5c0f\u5305\u541e\u5410\u91cf\u654f\u611f\uff0c\u9700\u8981\u5728\u76f8\u5e94\u73af\u5883\u4e0b \u8fdb\u884c\u6d4b\u8bd5\u5224\u65ad\u662f\u5426\u5f00\u542f eBPF \u4f18\u5316\u3002\u6211\u4eec\u4e5f\u4f1a\u540e\u7eed\u5bf9 eBPF TCP \u5c0f\u5305\u573a\u666f\u7684\u541e\u5410\u91cf\u8fdb\u884c\u4f18\u5316\u3002</p>"},{"location":"advance/accelerate-intra-node-tcp-with-ebpf/#_5","title":"\u53c2\u8003\u8d44\u6599","text":"<ol> <li>istio-tcpip-bypass</li> <li>Deep Dive TCP/IP Bypass with eBPF in Service Mesh</li> <li>Tanzu Service Mesh Acceleration using eBPF</li> </ol> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/cilium-hubble-observe/","title":"Cilium \u7f51\u7edc\u6d41\u91cf\u89c2\u6d4b","text":"<p>Kube-OVN \u5f53\u524d\u5df2\u7ecf\u652f\u6301\u4e0e Cilium \u96c6\u6210\uff0c\u5177\u4f53\u64cd\u4f5c\u53ef\u4ee5\u53c2\u8003 Cilium \u96c6\u6210\u3002</p> <p>Cilium \u63d0\u4f9b\u4e86\u4e30\u5bcc\u7684\u7f51\u7edc\u6d41\u91cf\u89c2\u6d4b\u80fd\u529b\uff0c\u6d41\u91cf\u53ef\u89c2\u6d4b\u6027\u662f\u7531 Hubble \u63d0\u4f9b\u7684\u3002Hubble \u53ef\u4ee5\u89c2\u5bdf\u8282\u70b9\u3001\u96c6\u7fa4\u751a\u81f3\u591a\u96c6\u7fa4\u573a\u666f\u4e0b\u8de8\u96c6\u7fa4\u7684\u6d41\u91cf\u3002</p>"},{"location":"advance/cilium-hubble-observe/#hubble","title":"\u5b89\u88c5 Hubble","text":"<p>\u9ed8\u8ba4\u7684 Cilium \u96c6\u6210\u5b89\u88c5\u4e2d\uff0c\u5e76\u6ca1\u6709\u5b89\u88c5 Hubble \u76f8\u5173\u7ec4\u4ef6\uff0c\u56e0\u6b64\u8981\u652f\u6301\u6d41\u91cf\u89c2\u6d4b\uff0c\u9700\u8981\u5148\u5728\u73af\u5883\u4e0a\u8865\u5145\u5b89\u88c5 Hubble\u3002</p> <p>\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u4f7f\u7528 helm \u5b89\u88c5 Hubble\uff1a</p> <pre><code>helm upgrade cilium cilium/cilium --version 1.11.6 \\\n--namespace kube-system \\\n--reuse-values \\\n--set hubble.relay.enabled=true \\\n--set hubble.ui.enabled=true\n</code></pre> <p>\u8865\u5145\u5b89\u88c5 Hubble \u4e4b\u540e\uff0c\u6267\u884c <code>cilium status</code> \u67e5\u770b\u7ec4\u4ef6\u72b6\u6001\uff0c\u786e\u8ba4 Hubble \u5b89\u88c5\u6210\u529f\u3002</p> <pre><code># cilium status\n/\u00af\u00af\\\n/\u00af\u00af\\__/\u00af\u00af\\    Cilium:         OK\n \\__/\u00af\u00af\\__/    Operator:       OK\n /\u00af\u00af\\__/\u00af\u00af\\    Hubble:         OK\n \\__/\u00af\u00af\\__/    ClusterMesh:    disabled\n    \\__/\n\nDeployment        hubble-relay       Desired: 1, Ready: 1/1, Available: 1/1\nDeployment        cilium-operator    Desired: 2, Ready: 2/2, Available: 2/2\nDaemonSet         cilium             Desired: 2, Ready: 2/2, Available: 2/2\nDeployment        hubble-ui          Desired: 1, Ready: 1/1, Available: 1/1\nContainers:       cilium             Running: 2\nhubble-ui          Running: 1\nhubble-relay       Running: 1\ncilium-operator    Running: 2\nCluster Pods:     16/17 managed by Cilium\nImage versions    hubble-relay       quay.io/cilium/hubble-relay:v1.11.6@sha256:fd9034a2d04d5b973f1e8ed44f230ea195b89c37955ff32e34e5aa68f3ed675a: 1\ncilium-operator    quay.io/cilium/operator-generic:v1.11.6@sha256:9f6063c7bcaede801a39315ec7c166309f6a6783e98665f6693939cf1701bc17: 2\ncilium             quay.io/cilium/cilium:v1.11.6@sha256:f7f93c26739b6641a3fa3d76b1e1605b15989f25d06625260099e01c8243f54c: 2\nhubble-ui          quay.io/cilium/hubble-ui:v0.9.0@sha256:0ef04e9a29212925da6bdfd0ba5b581765e41a01f1cc30563cef9b30b457fea0: 1\nhubble-ui          quay.io/cilium/hubble-ui-backend:v0.9.0@sha256:000df6b76719f607a9edefb9af94dfd1811a6f1b6a8a9c537cba90bf12df474b: 1\napple@bogon cilium %\n</code></pre> <p>\u5b89\u88c5 Hubble \u7ec4\u4ef6\u4e4b\u540e\uff0c\u9700\u8981\u5b89\u88c5\u547d\u4ee4\u884c\uff0c\u7528\u4e8e\u5728\u73af\u5883\u4e0a\u67e5\u770b\u6d41\u91cf\u4fe1\u606f\u3002 \u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u5b89\u88c5 Hubble CLI :</p> <pre><code>curl -L --fail --remote-name-all https://github.com/cilium/hubble/releases/download/v0.10.0/hubble-linux-amd64.tar.gz\nsudo tar xzvfC hubble-linux-amd64.tar.gz /usr/local/bin\n</code></pre>"},{"location":"advance/cilium-hubble-observe/#_1","title":"\u90e8\u7f72\u6d4b\u8bd5\u4e1a\u52a1","text":"<p>Cilium \u5b98\u65b9\u63d0\u4f9b\u4e86\u4e00\u4e2a\u6d41\u91cf\u6d4b\u8bd5\u7684\u90e8\u7f72\u65b9\u6848\uff0c\u53ef\u4ee5\u76f4\u63a5\u4f7f\u7528\u5b98\u65b9\u90e8\u7f72\u7684\u4e1a\u52a1\u8fdb\u884c\u6d4b\u8bd5\u3002</p> <p>\u6267\u884c\u547d\u4ee4 <code>cilium connectivity test</code>\uff0cCilium \u4f1a\u81ea\u52a8\u521b\u5efa <code>cilium-test</code> \u7684 Namespace\uff0c\u540c\u65f6\u5728 cilium-test \u4e0b\u90e8\u7f72\u6d4b\u8bd5\u4e1a\u52a1\u3002</p> <p>\u6b63\u5e38\u90e8\u7f72\u5b8c\u540e\uff0c\u53ef\u4ee5\u67e5\u770b cilium-test namespace \u4e0b\u7684\u8d44\u6e90\u4fe1\u606f\uff0c\u53c2\u8003\u5982\u4e0b\uff1a</p> <pre><code># kubectl get all -n cilium-test\nNAME                                  READY   STATUS    RESTARTS   AGE\npod/client-7df6cfbf7b-z5t2j           1/1     Running   0          21s\npod/client2-547996d7d8-nvgxg          1/1     Running   0          21s\npod/echo-other-node-d79544ccf-hl4gg   2/2     Running   0          21s\npod/echo-same-node-5d466d5444-ml7tc   2/2     Running   0          21s\n\nNAME                      TYPE       CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE\nservice/echo-other-node   NodePort   10.109.58.126   &lt;none&gt;        8080:32269/TCP   21s\nservice/echo-same-node    NodePort   10.108.70.32    &lt;none&gt;        8080:32490/TCP   21s\n\nNAME                              READY   UP-TO-DATE   AVAILABLE   AGE\ndeployment.apps/client            1/1     1            1           21s\ndeployment.apps/client2           1/1     1            1           21s\ndeployment.apps/echo-other-node   1/1     1            1           21s\ndeployment.apps/echo-same-node    1/1     1            1           21s\n\nNAME                                        DESIRED   CURRENT   READY   AGE\nreplicaset.apps/client-7df6cfbf7b           1         1         1       21s\nreplicaset.apps/client2-547996d7d8          1         1         1       21s\nreplicaset.apps/echo-other-node-d79544ccf   1         1         1       21s\nreplicaset.apps/echo-same-node-5d466d5444   1         1         1       21s\n</code></pre>"},{"location":"advance/cilium-hubble-observe/#_2","title":"\u4f7f\u7528\u547d\u4ee4\u884c\u8fdb\u884c\u6d41\u91cf\u89c2\u6d4b","text":"<p>\u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u7f51\u7edc\u6d41\u91cf\u89c2\u6d4b\u4ec5\u63d0\u4f9b\u6bcf\u4e2a\u8282\u70b9 Cilium \u4ee3\u7406\u89c2\u5bdf\u5230\u7684\u6d41\u91cf\u3002 \u53ef\u4ee5\u5728 <code>kube-system namespace</code> \u4e0b\u7684 Cilium \u4ee3\u7406 pod \u4e2d\u6267\u884c <code>hubble observe</code> \u547d\u4ee4\uff0c\u67e5\u770b\u8be5\u8282\u70b9\u4e0a\u7684\u6d41\u91cf\u4fe1\u606f\u3002</p> <pre><code># kubectl get pod -n kube-system -o wide\nNAME                                             READY   STATUS    RESTARTS   AGE     IP           NODE                     NOMINATED NODE   READINESS GATES\ncilium-d6h56                                     1/1     Running   0          2d20h   172.18.0.2   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;\ncilium-operator-5887f78bbb-c7sb2                 1/1     Running   0          2d20h   172.18.0.2   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;\ncilium-operator-5887f78bbb-wj8gt                 1/1     Running   0          2d20h   172.18.0.3   kube-ovn-control-plane   &lt;none&gt;           &lt;none&gt;\ncilium-tq5xb                                     1/1     Running   0          2d20h   172.18.0.3   kube-ovn-control-plane   &lt;none&gt;           &lt;none&gt;\nkube-ovn-pinger-7lgk8                            1/1     Running   0          21h     10.16.0.19   kube-ovn-control-plane   &lt;none&gt;           &lt;none&gt;\nkube-ovn-pinger-msvcn                            1/1     Running   0          21h     10.16.0.18   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;\n\n# kubectl exec -it -n kube-system cilium-d6h56 -- bash\nroot@kube-ovn-worker:/home/cilium# hubble observe --from-namespace kube-system\nJul 29 03:24:25.551: kube-system/kube-ovn-pinger-msvcn:35576 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: ACK, PSH)\nJul 29 03:24:25.561: kube-system/kube-ovn-pinger-msvcn:35576 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: RST)\nJul 29 03:24:25.561: kube-system/kube-ovn-pinger-msvcn:35576 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: ACK, RST)\nJul 29 03:24:25.572: kube-system/kube-ovn-pinger-msvcn:35578 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: SYN)\nJul 29 03:24:25.572: kube-system/kube-ovn-pinger-msvcn:35578 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: ACK)\nJul 29 03:24:25.651: kube-system/kube-ovn-pinger-msvcn:35578 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: ACK, PSH)\nJul 29 03:24:25.661: kube-system/kube-ovn-pinger-msvcn:35578 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: RST)\nJul 29 03:24:25.661: kube-system/kube-ovn-pinger-msvcn:35578 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: ACK, RST)\nJul 29 03:24:25.761: kube-system/kube-ovn-pinger-msvcn:52004 -&gt; 172.18.0.3:6443 to-stack FORWARDED (TCP Flags: ACK, PSH)\nJul 29 03:24:25.779: kube-system/kube-ovn-pinger-msvcn -&gt; kube-system/kube-ovn-pinger-7lgk8 to-stack FORWARDED (ICMPv4 EchoRequest)\nJul 29 03:24:25.779: kube-system/kube-ovn-pinger-msvcn &lt;- kube-system/kube-ovn-pinger-7lgk8 to-endpoint FORWARDED (ICMPv4 EchoReply)\nJul 29 03:24:25.866: kube-system/hubble-ui-7596f7ff6f-7j6f2:55836 &lt;- kube-system/hubble-relay-959988db5-zc5vv:4245 to-stack FORWARDED (TCP Flags: ACK)\nJul 29 03:24:25.866: kube-system/hubble-ui-7596f7ff6f-7j6f2:55836 &lt;- kube-system/hubble-relay-959988db5-zc5vv:80 to-endpoint FORWARDED (TCP Flags: ACK)\nJul 29 03:24:25.866: kube-system/hubble-ui-7596f7ff6f-7j6f2:55836 -&gt; kube-system/hubble-relay-959988db5-zc5vv:4245 to-stack FORWARDED (TCP Flags: ACK)\nJul 29 03:24:25.866: kube-system/hubble-ui-7596f7ff6f-7j6f2:55836 -&gt; kube-system/hubble-relay-959988db5-zc5vv:4245 to-endpoint FORWARDED (TCP Flags: ACK)\nJul 29 03:24:25.975: kube-system/kube-ovn-pinger-7lgk8 -&gt; kube-system/kube-ovn-pinger-msvcn to-endpoint FORWARDED (ICMPv4 EchoRequest)\nJul 29 03:24:25.975: kube-system/kube-ovn-pinger-7lgk8 &lt;- kube-system/kube-ovn-pinger-msvcn to-stack FORWARDED (ICMPv4 EchoReply)\nJul 29 03:24:25.979: kube-system/kube-ovn-pinger-msvcn -&gt; 172.18.0.3 to-stack FORWARDED (ICMPv4 EchoRequest)\nJul 29 03:24:26.037: kube-system/coredns-6d4b75cb6d-lbgjg:36430 -&gt; 172.18.0.3:6443 to-stack FORWARDED (TCP Flags: ACK)\nJul 29 03:24:26.282: kube-system/kube-ovn-pinger-msvcn -&gt; 172.18.0.2 to-stack FORWARDED (ICMPv4 EchoRequest)\n</code></pre> <p>\u90e8\u7f72 Hubble Relay \u540e\uff0cHubble \u53ef\u4ee5\u63d0\u4f9b\u5b8c\u6574\u7684\u96c6\u7fa4\u8303\u56f4\u7684\u7f51\u7edc\u6d41\u91cf\u89c2\u6d4b\u3002</p>"},{"location":"advance/cilium-hubble-observe/#_3","title":"\u914d\u7f6e\u7aef\u53e3\u8f6c\u53d1","text":"<p>\u4e3a\u4e86\u80fd\u6b63\u5e38\u8bbf\u95ee Hubble API\uff0c\u9700\u8981\u521b\u5efa\u7aef\u53e3\u8f6c\u53d1\uff0c\u5c06\u672c\u5730\u8bf7\u6c42\u8f6c\u53d1\u5230 Hubble Service\u3002\u53ef\u4ee5\u6267\u884c <code>kubectl port-forward deployment/hubble-relay -n kube-system 4245:4245</code> \u547d\u4ee4\uff0c\u5728\u5f53\u524d\u7ec8\u7aef\u5f00\u542f\u7aef\u53e3\u8f6c\u53d1\u3002</p> <p>\u7aef\u53e3\u8f6c\u53d1\u914d\u7f6e\u53ef\u4ee5\u53c2\u8003 \u7aef\u53e3\u8f6c\u53d1\u3002</p> <p><code>kubectl port-forward</code> \u547d\u4ee4\u4e0d\u4f1a\u8fd4\u56de\uff0c\u9700\u8981\u6253\u5f00\u53e6\u4e00\u4e2a\u7ec8\u7aef\u6765\u7ee7\u7eed\u6d4b\u8bd5\u3002</p> <p>\u914d\u7f6e\u5b8c\u7aef\u53e3\u8f6c\u53d1\u4e4b\u540e\uff0c\u5728\u7ec8\u7aef\u6267\u884c <code>hubble status</code> \u547d\u4ee4\uff0c\u5982\u679c\u6709\u7c7b\u4f3c\u5982\u4e0b\u8f93\u51fa\uff0c\u5219\u7aef\u53e3\u8f6c\u53d1\u914d\u7f6e\u6b63\u786e\uff0c\u53ef\u4ee5\u4f7f\u7528\u547d\u4ee4\u884c\u8fdb\u884c\u6d41\u91cf\u89c2\u6d4b\u3002</p> <pre><code># hubble status\nHealthcheck (via localhost:4245): Ok\nCurrent/Max Flows: 8,190/8,190 (100.00%)\nFlows/s: 22.86\nConnected Nodes: 2/2\n</code></pre>"},{"location":"advance/cilium-hubble-observe/#_4","title":"\u547d\u4ee4\u884c\u89c2\u6d4b","text":"<p>\u5728\u7ec8\u7aef\u4e0a\u6267\u884c <code>hubble observe</code> \u547d\u4ee4\uff0c\u67e5\u770b\u96c6\u7fa4\u7684\u6d41\u91cf\u4fe1\u606f\u3002 \u89c2\u6d4b\u5230\u7684 cilium-test \u76f8\u5173\u7684\u6d4b\u8bd5\u6d41\u91cf\u53c2\u8003\u5982\u4e0b\uff1a </p> <p>\u9700\u8981\u6ce8\u610f\u7684\u662f\uff0c <code>hubble observe</code> \u547d\u4ee4\u7684\u663e\u793a\u7ed3\u679c\uff0c\u662f\u5f53\u524d\u547d\u4ee4\u884c\u6267\u884c\u65f6\u67e5\u8be2\u5230\u7684\u6d41\u91cf\u4fe1\u606f\u3002\u591a\u6b21\u6267\u884c\u547d\u4ee4\u884c\uff0c\u53ef\u4ee5\u67e5\u770b\u5230\u4e0d\u540c\u7684\u6d41\u91cf\u4fe1\u606f\u3002 \u66f4\u591a\u8be6\u7ec6\u7684\u89c2\u6d4b\u4fe1\u606f\uff0c\u53ef\u4ee5\u6267\u884c <code>hubble help observe</code> \u547d\u4ee4\u67e5\u770b Hubble CLI \u7684\u8be6\u7ec6\u4f7f\u7528\u65b9\u5f0f\u3002</p>"},{"location":"advance/cilium-hubble-observe/#ui","title":"\u4f7f\u7528 UI \u8fdb\u884c\u6d41\u91cf\u89c2\u6d4b","text":"<p>\u6267\u884c <code>cilium status</code> \u547d\u4ee4\uff0c\u786e\u8ba4 Hubble UI \u5df2\u7ecf\u5b89\u88c5\u6210\u529f\u3002\u5728\u7b2c\u4e8c\u6b65\u7684 Hubble \u5b89\u88c5\u4e2d\uff0c\u5df2\u7ecf\u8865\u5145\u4e86 UI \u7684\u5b89\u88c5\u3002</p> <p>\u6267\u884c\u547d\u4ee4 <code>cilium hubble ui</code> \u53ef\u4ee5\u81ea\u52a8\u521b\u5efa\u7aef\u53e3\u8f6c\u53d1\uff0c\u5c06 <code>hubble-ui service</code> \u6620\u5c04\u5230\u672c\u5730\u7aef\u53e3\u3002 \u6b63\u5e38\u60c5\u51b5\u4e0b\uff0c\u6267\u884c\u5b8c\u547d\u4ee4\u540e\uff0c\u4f1a\u81ea\u52a8\u6253\u5f00\u672c\u5730\u7684\u6d4f\u89c8\u5668\uff0c\u8df3\u8f6c\u5230 Hubble UI \u754c\u9762\u3002\u5982\u679c\u6ca1\u6709\u81ea\u52a8\u8df3\u8f6c\uff0c\u5728\u6d4f\u89c8\u5668\u4e2d\u8f93\u5165 <code>http://localhost:12000</code> \u6253\u5f00 UI \u89c2\u5bdf\u754c\u9762\u3002</p> <p>\u5728\u754c\u9762\u5de6\u4e0a\u89d2\uff0c\u9009\u62e9 <code>cilium-test</code> namespace\uff0c\u67e5\u770b Cilium \u63d0\u4f9b\u7684\u6d4b\u8bd5\u6d41\u91cf\u4fe1\u606f\u3002 </p>"},{"location":"advance/cilium-hubble-observe/#hubble_1","title":"Hubble \u6d41\u91cf\u76d1\u63a7","text":"<p>Hubble \u7ec4\u4ef6\u63d0\u4f9b\u4e86\u96c6\u7fa4\u4e2d Pod \u7f51\u7edc\u884c\u4e3a\u7684\u76d1\u63a7\uff0c\u4e3a\u4e86\u652f\u6301\u67e5\u770b Hubble \u63d0\u4f9b\u7684\u76d1\u63a7\u6570\u636e\uff0c\u9700\u8981\u4f7f\u80fd\u76d1\u63a7\u7edf\u8ba1\u3002</p> <p>\u53c2\u8003\u4ee5\u4e0b\u547d\u4ee4\uff0c\u8865\u5145 <code>hubble.metrics.enabled</code> \u914d\u7f6e\u9879:</p> <pre><code>helm upgrade cilium cilium/cilium --version 1.11.6 \\\n--namespace kube-system \\\n--reuse-values \\\n--set hubble.relay.enabled=true \\\n--set hubble.ui.enabled=true \\\n--set hubble.metrics.enabled=\"{dns,drop,tcp,flow,icmp,http}\"\n</code></pre> <p>\u90e8\u7f72\u4e4b\u540e\uff0c\u4f1a\u5728 kube-system namespace \u751f\u6210\u540d\u79f0\u4e3a <code>hubble-metrics</code> \u7684\u670d\u52a1\u3002\u901a\u8fc7\u8bbf\u95ee Endpoints \u67e5\u8be2 Hubble \u63d0\u4f9b\u7684\u76d1\u63a7\u6307\u6807\uff0c\u53c2\u8003\u5982\u4e0b:</p> <pre><code># curl 172.18.0.2:9091/metrics\n# HELP hubble_drop_total Number of drops\n# TYPE hubble_drop_total counter\nhubble_drop_total{protocol=\"ICMPv6\",reason=\"Unsupported L3 protocol\"} 2\n# HELP hubble_flows_processed_total Total number of flows processed\n# TYPE hubble_flows_processed_total counter\nhubble_flows_processed_total{protocol=\"ICMPv4\",subtype=\"to-endpoint\",type=\"Trace\",verdict=\"FORWARDED\"} 335\nhubble_flows_processed_total{protocol=\"ICMPv4\",subtype=\"to-stack\",type=\"Trace\",verdict=\"FORWARDED\"} 335\nhubble_flows_processed_total{protocol=\"ICMPv6\",subtype=\"\",type=\"Drop\",verdict=\"DROPPED\"} 2\nhubble_flows_processed_total{protocol=\"TCP\",subtype=\"to-endpoint\",type=\"Trace\",verdict=\"FORWARDED\"} 8282\nhubble_flows_processed_total{protocol=\"TCP\",subtype=\"to-stack\",type=\"Trace\",verdict=\"FORWARDED\"} 6767\nhubble_flows_processed_total{protocol=\"UDP\",subtype=\"to-endpoint\",type=\"Trace\",verdict=\"FORWARDED\"} 1642\nhubble_flows_processed_total{protocol=\"UDP\",subtype=\"to-stack\",type=\"Trace\",verdict=\"FORWARDED\"} 1642\n# HELP hubble_icmp_total Number of ICMP messages\n# TYPE hubble_icmp_total counter\nhubble_icmp_total{family=\"IPv4\",type=\"EchoReply\"} 335\nhubble_icmp_total{family=\"IPv4\",type=\"EchoRequest\"} 335\nhubble_icmp_total{family=\"IPv4\",type=\"RouterSolicitation\"} 2\n# HELP hubble_tcp_flags_total TCP flag occurrences\n# TYPE hubble_tcp_flags_total counter\nhubble_tcp_flags_total{family=\"IPv4\",flag=\"FIN\"} 2043\nhubble_tcp_flags_total{family=\"IPv4\",flag=\"RST\"} 301\nhubble_tcp_flags_total{family=\"IPv4\",flag=\"SYN\"} 1169\nhubble_tcp_flags_total{family=\"IPv4\",flag=\"SYN-ACK\"} 1169\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/cilium-networkpolicy/","title":"Cilium NetworkPolicy \u652f\u6301","text":"<p>Kube-OVN \u5f53\u524d\u5df2\u7ecf\u652f\u6301\u4e0e Cilium \u96c6\u6210\uff0c\u5177\u4f53\u64cd\u4f5c\u53ef\u4ee5\u53c2\u8003 Cilium \u96c6\u6210\u3002</p> <p>\u5728\u96c6\u6210 Cilium \u4e4b\u540e\uff0c\u5c31\u53ef\u4ee5\u4f7f\u7528 Cilium \u4f18\u79c0\u7684\u7f51\u7edc\u7b56\u7565\u80fd\u529b\uff0c\u5b9e\u73b0\u5bf9\u6d41\u91cf\u8bbf\u95ee\u7684\u63a7\u5236\u3002\u4ee5\u4e0b\u6587\u6863\u63d0\u4f9b\u4e86\u5bf9 Cilium L3 \u548c L4 \u7f51\u7edc\u7b56\u7565\u80fd\u529b\u7684\u96c6\u6210\u9a8c\u8bc1\u3002</p>"},{"location":"advance/cilium-networkpolicy/#_1","title":"\u9a8c\u8bc1\u6b65\u9aa4","text":""},{"location":"advance/cilium-networkpolicy/#pod","title":"\u521b\u5efa\u6d4b\u8bd5 Pod","text":"<p>\u521b\u5efa namespace <code>test</code>\u3002\u53c2\u8003\u4ee5\u4e0b yaml\uff0c\u5728 test namespace \u4e2d\u521b\u5efa\u6307\u5b9a label <code>app=test</code> \u7684 Pod\uff0c\u4f5c\u4e3a\u6d4b\u8bd5\u8bbf\u95ee\u7684\u76ee\u7684 Pod\u3002</p> <pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nlabels:\napp: test\nname: test\nnamespace: test\nspec:\nreplicas: 1\nselector:\nmatchLabels:\napp: test\nstrategy:\nrollingUpdate:\nmaxSurge: 25%\nmaxUnavailable: 25%\ntype: RollingUpdate\ntemplate:\nmetadata:\nlabels:\napp: test\nspec:\ncontainers:\n- image: docker.io/library/nginx:alpine\nimagePullPolicy: IfNotPresent\nname: nginx\n</code></pre> <p>\u540c\u6837\u53c2\u8003\u4ee5\u4e0b yaml\uff0c\u5728 default namespace \u4e0b\u521b\u5efa\u6307\u5b9a label <code>app=dynamic</code> \u7684 Pod \u4e3a\u53d1\u8d77\u8bbf\u95ee\u6d4b\u8bd5\u7684 Pod\u3002</p> <pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nlabels:\napp: dynamic\nname: dynamic\nnamespace: default\nspec:\nreplicas: 2\nselector:\nmatchLabels:\napp: dynamic\nstrategy:\nrollingUpdate:\nmaxSurge: 25%\nmaxUnavailable: 25%\ntype: RollingUpdate\ntemplate:\nmetadata:\ncreationTimestamp: null\nlabels:\napp: dynamic\nspec:\ncontainers:\n- image: docker.io/library/nginx:alpine\nimagePullPolicy: IfNotPresent\nname: nginx\n</code></pre> <p>\u67e5\u770b\u6d4b\u8bd5 Pod \u4ee5\u53ca Label \u4fe1\u606f:</p> <pre><code># kubectl get pod -o wide --show-labels\nNAME                         READY   STATUS    RESTARTS   AGE   IP           NODE                     NOMINATED NODE   READINESS GATES   LABELS\ndynamic-7d8d7874f5-9v5c4     1/1     Running   0          28h   10.16.0.35   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;            app=dynamic,pod-template-hash=7d8d7874f5\ndynamic-7d8d7874f5-s8z2n     1/1     Running   0          28h   10.16.0.36   kube-ovn-control-plane   &lt;none&gt;           &lt;none&gt;            app=dynamic,pod-template-hash=7d8d7874f5\n# kubectl get pod -o wide -n test --show-labels\nNAME                           READY   STATUS    RESTARTS   AGE     IP           NODE                     NOMINATED NODE   READINESS GATES   LABELS\ndynamic-7d8d7874f5-6dsg6       1/1     Running   0          7h20m   10.16.0.2    kube-ovn-control-plane   &lt;none&gt;           &lt;none&gt;            app=dynamic,pod-template-hash=7d8d7874f5\ndynamic-7d8d7874f5-tjgtp       1/1     Running   0          7h46m   10.16.0.42   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;            app=dynamic,pod-template-hash=7d8d7874f5\nlabel-test1-77b6764857-swq4k   1/1     Running   0          3h43m   10.16.0.12   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;            app=test1,pod-template-hash=77b6764857\n\n// \u4ee5\u4e0b\u4e3a\u6d4b\u8bd5\u8bbf\u95ee\u76ee\u7684 Pod\ntest-54c98bc466-mft5s          1/1     Running   0          8h      10.16.0.41   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;            app=test,pod-template-hash=54c98bc466\n</code></pre>"},{"location":"advance/cilium-networkpolicy/#l3","title":"L3 \u7f51\u7edc\u7b56\u7565\u6d4b\u8bd5","text":"<p>\u53c2\u8003\u4ee5\u4e0b yaml\uff0c\u521b\u5efa <code>CiliumNetworkPolicy</code> \u8d44\u6e90:</p> <pre><code>apiVersion: \"cilium.io/v2\"\nkind: CiliumNetworkPolicy\nmetadata:\nname: \"l3-rule\"\nnamespace: test\nspec:\nendpointSelector:\nmatchLabels:\napp: test\ningress:\n- fromEndpoints:\n- matchLabels:\napp: dynamic\n</code></pre> <p>\u5728 default namespace \u4e0b\u7684\u6d4b\u8bd5 Pod \u4e2d\uff0c\u53d1\u8d77\u5bf9\u76ee\u7684 Pod \u7684\u8bbf\u95ee\uff0c\u7ed3\u679c\u8bbf\u95ee\u4e0d\u901a\u3002 \u4f46\u662f\u5728 test namespace \u4e0b\uff0c\u6d4b\u8bd5\u5230\u76ee\u7684 Pod \u7684\u8bbf\u95ee\uff0c\u6d4b\u8bd5\u6b63\u5e38\u3002</p> <p>default namespace \u4e0b\u6d4b\u8bd5\u7ed3\u679c:</p> <pre><code># kubectl exec -it dynamic-7d8d7874f5-9v5c4 -- bash\nbash-5.0# ping -c 3 10.16.0.41\nPING 10.16.0.41 (10.16.0.41): 56 data bytes\n\n--- 10.16.0.41 ping statistics ---\n3 packets transmitted, 0 packets received, 100% packet loss\n</code></pre> <p>test namepsace \u4e0b Pod \u7684\u6d4b\u8bd5\uff0c\u8bbf\u95ee\u6b63\u5e38:</p> <pre><code># kubectl exec -it -n test dynamic-7d8d7874f5-6dsg6 -- bash\nbash-5.0# ping -c 3 10.16.0.41\nPING 10.16.0.41 (10.16.0.41): 56 data bytes\n64 bytes from 10.16.0.41: seq=0 ttl=64 time=2.558 ms\n64 bytes from 10.16.0.41: seq=1 ttl=64 time=0.223 ms\n64 bytes from 10.16.0.41: seq=2 ttl=64 time=0.304 ms\n\n--- 10.16.0.41 ping statistics ---\n3 packets transmitted, 3 packets received, 0% packet loss\nround-trip min/avg/max = 0.223/1.028/2.558 ms\n</code></pre> <p>\u67e5\u770b Cilium \u5b98\u65b9\u6587\u6863\u89e3\u91ca\uff0c<code>CiliumNetworkPolicy</code> \u8d44\u6e90\u5c06\u9650\u5236\u63a7\u5236\u5728\u4e86 <code>Namespace</code> \u7ea7\u522b\u3002\u8be6\u7ec6\u4fe1\u606f\u53ef\u4ee5\u67e5\u770b Cilium \u9650\u5236\u3002</p> <p>\u5728\u6709\u7f51\u7edc\u7b56\u7565\u89c4\u5219\u5339\u914d\u7684\u60c5\u51b5\u4e0b\uff0c\u53ea\u6709\u540c\u4e00\u4e2a Namespace \u7684 Pod \uff0c\u624d\u53ef\u4ee5\u6309\u7167\u89c4\u5219\u8fdb\u884c\u8bbf\u95ee\uff0c\u9ed8\u8ba4\u62d2\u7edd\u5176\u4ed6 Namespace \u7684 Pod \u8fdb\u884c\u8bbf\u95ee\u3002</p> <p>\u5982\u679c\u60f3\u5b9e\u73b0\u8de8 Namespace \u7684\u8bbf\u95ee\uff0c\u9700\u8981\u5728\u89c4\u5219\u4e2d\u660e\u786e\u6307\u5b9a Namespace \u4fe1\u606f\u3002</p> <p>\u53c2\u8003\u6587\u6863\uff0c\u4fee\u6539 <code>CiliumNetworkPolicy</code> \u8d44\u6e90\uff0c\u589e\u52a0 namespace \u4fe1\u606f:</p> <pre><code>  ingress:\n- fromEndpoints:\n- matchLabels:\napp: dynamic\nk8s:io.kubernetes.pod.namespace: default    // \u63a7\u5236\u5176\u4ed6 Namespace \u4e0b\u7684 Pod \u8bbf\u95ee\n</code></pre> <p>\u67e5\u770b\u4fee\u6539\u540e\u7684 <code>CiliumNetworkPolicy</code> \u8d44\u6e90\u4fe1\u606f:</p> <pre><code># kubectl get cnp -n test  -o yaml l3-rule\napiVersion: cilium.io/v2\nkind: CiliumNetworkPolicy\nmetadata:\n  name: l3-rule\n  namespace: test\nspec:\n  endpointSelector:\n    matchLabels:\n      app: test\ningress:\n  - fromEndpoints:\n    - matchLabels:\n        app: dynamic\n    - matchLabels:\n        app: dynamic\n        k8s:io.kubernetes.pod.namespace: default\n</code></pre> <p>\u518d\u6b21\u6d4b\u8bd5 default namespace \u4e0b\u7684 Pod \u8bbf\u95ee\uff0c\u76ee\u7684 Pod \u8bbf\u95ee\u6b63\u5e38:</p> <pre><code># kubectl exec -it dynamic-7d8d7874f5-9v5c4 -n test -- bash\nbash-5.0# ping -c 3 10.16.0.41\nPING 10.16.0.41 (10.16.0.41): 56 data bytes\n64 bytes from 10.16.0.41: seq=0 ttl=64 time=2.383 ms\n64 bytes from 10.16.0.41: seq=1 ttl=64 time=0.115 ms\n64 bytes from 10.16.0.41: seq=2 ttl=64 time=0.142 ms\n\n--- 10.16.0.41 ping statistics ---\n3 packets transmitted, 3 packets received, 0% packet loss\nround-trip min/avg/max = 0.115/0.880/2.383 ms\n</code></pre> <p>\u4f7f\u7528\u6807\u51c6\u7684 Kubernetes \u7f51\u7edc\u7b56\u7565 networkpolicy\uff0c\u6d4b\u8bd5\u7ed3\u679c\u663e\u793a Cilium \u540c\u6837\u5c06\u8bbf\u95ee\u9650\u5236\u5728\u540c\u4e00\u4e2a Namespace \u5185\uff0c\u8de8 Namespace \u7684\u8bbf\u95ee\u662f\u7981\u6b62\u7684\u3002</p> <p>\u8fd9\u70b9\u4e0e Kube-OVN \u5b9e\u73b0\u662f\u4e0d\u540c\u7684\u3002Kube-OVN \u652f\u6301\u6807\u51c6\u7684 k8s \u7f51\u7edc\u7b56\u7565\uff0c\u9650\u5236\u4e86\u5177\u4f53 Namespace \u4e0b\u7684\u76ee\u7684 Pod\uff0c\u4f46\u662f\u5bf9\u6e90\u5730\u5740 Pod\uff0c\u662f\u6ca1\u6709 Namespace \u9650\u5236\u7684\uff0c\u4efb\u4f55 Namespace \u4e0b\u7b26\u5408\u9650\u5236\u89c4\u5219\u7684 Pod\uff0c\u90fd\u53ef\u4ee5\u5b9e\u73b0\u5bf9\u76ee\u7684 Pod \u7684\u8bbf\u95ee\u3002</p>"},{"location":"advance/cilium-networkpolicy/#l4","title":"L4 \u7f51\u7edc\u7b56\u7565\u6d4b\u8bd5","text":"<p>\u53c2\u8003\u4ee5\u4e0b yaml\uff0c\u521b\u5efa L4 \u5c42\u7684\u7f51\u7edc\u7b56\u7565\u8d44\u6e90:</p> <pre><code>apiVersion: \"cilium.io/v2\"\nkind: CiliumNetworkPolicy\nmetadata:\nname: \"l4-rule\"\nnamespace: test\nspec:\nendpointSelector:\nmatchLabels:\napp: test\ningress:\n- fromEndpoints:\n- matchLabels:\napp: dynamic\ntoPorts:\n- ports:\n- port: \"80\"\nprotocol: TCP\n</code></pre> <p>\u6d4b\u8bd5\u76f8\u540c Namespace \u4e0b\uff0c\u7b26\u5408\u7f51\u7edc\u7b56\u7565\u89c4\u5219\u7684 Pod \u7684\u8bbf\u95ee</p> <pre><code># kubectl exec -it -n test dynamic-7d8d7874f5-6dsg6 -- bash\nbash-5.0# ping -c 3 10.16.0.41\nPING 10.16.0.41 (10.16.0.41): 56 data bytes\n\n--- 10.16.0.41 ping statistics ---\n3 packets transmitted, 0 packets received, 100% packet loss\nbash-5.0#\nbash-5.0# curl 10.16.0.41:80\n&lt;html&gt;\n&lt;head&gt;\n        &lt;title&gt;Hello World!&lt;/title&gt;\n        &lt;link href='//fonts.googleapis.com/css?family=Open+Sans:400,700' rel='stylesheet' type='text/css'&gt;\n        &lt;style&gt;\n        body {\nbackground-color: white;\ntext-align: center;\npadding: 50px;\nfont-family: \"Open Sans\",\"Helvetica Neue\",Helvetica,Arial,sans-serif;\n}\n#logo {\nmargin-bottom: 40px;\n}\n&lt;/style&gt;\n&lt;/head&gt;\n&lt;body&gt;\n                &lt;h1&gt;Hello World!&lt;/h1&gt;\n                                &lt;h3&gt;Links found&lt;/h3&gt;\n        &lt;h3&gt;I am on  test-54c98bc466-mft5s&lt;/h3&gt;\n        &lt;h3&gt;Cookie                  =&lt;/h3&gt;\n                                        &lt;b&gt;KUBERNETES&lt;/b&gt; listening in 443 available at tcp://10.96.0.1:443&lt;br /&gt;\n                                                &lt;h3&gt;my name is hanhouchao!&lt;/h3&gt;\n                        &lt;h3&gt; RequestURI='/'&lt;/h3&gt;\n&lt;/body&gt;\n&lt;/html&gt;\n</code></pre> <p>\u76f8\u540c Namespace \u4e0b\uff0c\u4e0d\u7b26\u5408\u7f51\u7edc\u7b56\u7565\u89c4\u5219\u7684 Pod \u8bbf\u95ee\u6d4b\u8bd5</p> <pre><code># kubectl exec -it -n test label-test1-77b6764857-swq4k -- bash\nbash-5.0# ping -c 3 10.16.0.41\nPING 10.16.0.41 (10.16.0.41): 56 data bytes\n\n--- 10.16.0.41 ping statistics ---\n3 packets transmitted, 0 packets received, 100% packet loss\nbash-5.0#\nbash-5.0# curl -v 10.16.0.41:80 --connect-timeout 10\n*   Trying 10.16.0.41:80...\n* After 10000ms connect time, move on!\n* connect to 10.16.0.41 port 80 failed: Operation timed out\n* Connection timeout after 10001 ms\n* Closing connection 0\ncurl: (28) Connection timeout after 10001 ms\n</code></pre> <p>\u7f51\u7edc\u7b56\u7565\u751f\u6548\u540e\uff0c\u8de8 Namespace \u7684\u8bbf\u95ee\uff0c\u4f9d\u7136\u662f\u88ab\u7981\u6b62\u7684\uff0c\u8ddf L3 \u7f51\u7edc\u7b56\u7565\u6d4b\u8bd5\u7ed3\u679c\u4e00\u81f4\u3002</p> <p>\u5728 L4 \u7f51\u7edc\u89c4\u5219\u751f\u6548\u540e\uff0cping \u65e0\u6cd5\u4f7f\u7528\uff0c\u4f46\u662f\u7b26\u5408\u7b56\u7565\u89c4\u5219\u7684 TCP \u8bbf\u95ee\uff0c\u662f\u53ef\u4ee5\u6b63\u5e38\u6267\u884c\u7684\u3002</p> <p>\u5173\u4e8e ICMP \u7684\u9650\u5236\uff0c\u53ef\u4ee5\u53c2\u8003\u5b98\u65b9\u8bf4\u660e L4 \u9650\u5236\u8bf4\u660e\u3002</p>"},{"location":"advance/cilium-networkpolicy/#l7","title":"L7 \u7f51\u7edc\u7b56\u7565\u6d4b\u8bd5","text":"<p>chaining \u6a21\u5f0f\u4e0b\uff0cL7 \u7f51\u7edc\u7b56\u7565\u76ee\u524d\u662f\u5b58\u5728\u95ee\u9898\u7684\u3002\u5728 Cilium \u5b98\u65b9\u6587\u6863\u4e2d\uff0c\u5bf9\u8fd9\u79cd\u60c5\u51b5\u7ed9\u51fa\u4e86\u8bf4\u660e\uff0c\u53c2\u8003 Generic Veth Chaining\u3002</p> <p>\u8fd9\u4e2a\u95ee\u9898\u4f7f\u7528 issue 12454 \u8ddf\u8e2a\uff0c\u76ee\u524d\u8fd8\u6ca1\u6709\u89e3\u51b3\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/dhcp/","title":"DHCP \u8bbe\u7f6e","text":"<p>\u5728\u4f7f\u7528 SR-IOV \u6216 DPDK \u7c7b\u578b\u7f51\u7edc\u65f6\uff0cKubeVirt \u5185\u7f6e\u7684 DHCP \u65e0\u6cd5\u5728\u8be5\u7f51\u7edc\u6a21\u5f0f\u4e0b\u5de5\u4f5c\u3002Kube-OVN \u53ef\u4ee5\u5229\u7528 OVN \u7684 DHCP \u80fd\u529b\u5728\u5b50\u7f51\u7ea7\u522b\u8bbe\u7f6e  DHCP \u9009\u9879\uff0c\u4ece\u800c\u5e2e\u52a9\u8be5\u7f51\u7edc\u7c7b\u578b\u7684 KubeVirt \u865a\u673a\u6b63\u5e38\u4f7f\u7528 DHCP \u83b7\u5f97\u5206\u914d\u7684 IP \u5730\u5740\u3002Kube-OVN \u540c\u65f6\u652f\u6301 DHCPv4 \u548c DHCPv6\u3002</p> <p>\u5b50\u7f51 DHCP \u7684\u914d\u7f6e\u5982\u4e0b\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: sn-dual\nspec:\ncidrBlock: \"10.0.0.0/24,240e::a00/120\"\ndefault: false\ndisableGatewayCheck: true\ndisableInterConnection: false\nexcludeIps:\n- 10.0.0.1\n- 240e::a01\ngateway: 10.0.0.1,240e::a01\ngatewayNode: ''\ngatewayType: distributed\nnatOutgoing: false\nprivate: false\nprotocol: Dual\nprovider: ovn\nvpc: vpc-test\nenableDHCP: true\ndhcpV4Options: \"lease_time=3600,router=10.0.0.1,server_id=169.254.0.254,server_mac=00:00:00:2E:2F:B8\"\ndhcpV6Options: \"server_id=00:00:00:2E:2F:C5\"\nenableIPv6RA: true\nipv6RAConfigs: \"address_mode=dhcpv6_stateful,max_interval=30,min_interval=5,send_periodic=true\"\n</code></pre> <ul> <li><code>enableDHCP</code>: \u662f\u5426\u5f00\u542f\u5b50\u7f51\u7684 DHCP \u529f\u80fd\u3002</li> <li><code>dhcpV4Options</code>,<code>dhcpV6Options</code>: \u8be5\u5b57\u6bb5\u76f4\u63a5\u66b4\u9732 ovn-nb \u5185 DHCP \u76f8\u5173\u9009\u9879\uff0c\u8bf7\u53c2\u8003 DHCP Options\u3002 \u9ed8\u8ba4\u503c\u5206\u522b\u4e3a <code>\"lease_time=3600, router=$ipv4_gateway, server_id=169.254.0.254, server_mac=$random_mac\"</code> \u548c <code>server_id=$random_mac</code>\u3002</li> <li><code>enableIPv6RA</code>: \u662f\u5426\u5f00\u542f DHCPv6 \u7684\u8def\u7531\u5e7f\u64ad\u529f\u80fd\u3002</li> <li><code>ipv6RAConfigs</code>\uff1a\u8be5\u5b57\u6bb5\u76f4\u63a5\u66b4\u9732 ovn-nb \u5185 Logical_Router_Port \u76f8\u5173\u9009\u9879\uff0c\u8bf7\u53c2\u8003 Logical Router Port \u9ed8\u8ba4\u503c\u4e3a <code>address_mode=dhcpv6_stateful, max_interval=30, min_interval=5, send_periodic=true</code>\u3002</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/dpdk/","title":"DPDK \u652f\u6301","text":"<p>\u8be5\u6587\u6863\u4ecb\u7ecd Kube-OVN \u5982\u4f55\u548c OVS-DPDK \u7ed3\u5408\uff0c\u7ed9 KubeVirt \u7684\u865a\u673a\u63d0\u4f9b DPDK \u7c7b\u578b\u7684\u7f51\u7edc\u63a5\u53e3\u3002</p> <p>\u4e0a\u6e38\u7684 KubeVirt \u76ee\u524d\u8fd8\u672a\u652f\u6301 OVS-DPDK\uff0c\u7528\u6237\u9700\u8981\u81ea\u5df1\u901a\u8fc7\u76f8\u5173 patch Vhostuser implementation \u6784\u5efa KubeVirt \u6216 KVM Device Plugin \u6765\u4f7f\u7528 OVS-DPDK\u3002</p>"},{"location":"advance/dpdk/#_1","title":"\u524d\u63d0\u6761\u4ef6","text":"<ul> <li>\u8282\u70b9\u9700\u63d0\u4f9b\u4e13\u95e8\u7ed9 DPDK \u9a71\u52a8\u8fd0\u884c\u7684\u7f51\u5361\u3002</li> <li>\u8282\u70b9\u9700\u5f00\u542f Hugepages\u3002</li> </ul>"},{"location":"advance/dpdk/#dpdk_1","title":"\u7f51\u5361\u8bbe\u7f6e DPDK \u9a71\u52a8","text":"<p>\u8fd9\u91cc\u6211\u4eec\u4f7f\u7528 <code>driverctl</code> \u4e3a\u4f8b\u8fdb\u884c\u64cd\u4f5c\uff0c\u5177\u4f53\u53c2\u6570\u548c\u5176\u4ed6\u9a71\u52a8\u4f7f\u7528\u8bf7\u53c2\u8003 DPDK \u6587\u6863\u8fdb\u884c\u64cd\u4f5c\u3002</p> <pre><code>driverctl set-override 0000:00:0b.0 uio_pci_generic\n</code></pre>"},{"location":"advance/dpdk/#_2","title":"\u8282\u70b9\u914d\u7f6e","text":"<p>\u5bf9\u652f\u6301 OVS-DPDK \u7684\u8282\u70b9\u6253\u6807\u7b7e\uff0c\u4ee5\u4fbf Kube-OVN \u8fdb\u884c\u8bc6\u522b\u5904\u7406\uff1a</p> <pre><code>kubectl label nodes &lt;node&gt; ovn.kubernetes.io/ovs_dp_type=\"userspace\"\n</code></pre> <p>\u5728\u652f\u6301 OVS-DPDK \u8282\u70b9\u7684 <code>/opt/ovs-config</code> \u76ee\u5f55\u4e0b\u521b\u5efa\u914d\u7f6e\u6587\u4ef6 <code>ovs-dpdk-config</code>\uff1a</p> <pre><code>ENCAP_IP=192.168.122.193/24\nDPDK_DEV=0000:00:0b.0\n</code></pre> <ul> <li><code>ENCAP_IP</code>: \u96a7\u9053\u7aef\u70b9\u5730\u5740\u3002</li> <li><code>DPDK_DEV</code>: \u8bbe\u5907\u7684 PCI ID\u3002</li> </ul>"},{"location":"advance/dpdk/#kube-ovn","title":"\u5b89\u88c5 Kube-OVN","text":"<p>\u4e0b\u8f7d\u5b89\u88c5\u811a\u672c\uff1a</p> <pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/images/install.sh\n</code></pre> <p>\u542f\u7528 DPDK \u5b89\u88c5\u9009\u9879\u8fdb\u884c\u5b89\u88c5\uff1a</p> <pre><code>bash install.sh --with-hybrid-dpdk\n</code></pre>"},{"location":"advance/dpdk/#_3","title":"\u4f7f\u7528\u65b9\u5f0f","text":"<p>\u8fd9\u91cc\u6211\u4eec\u901a\u8fc7\u521b\u5efa\u4e00\u4e2a\u4f7f\u7528 vhostuser \u7c7b\u578b\u7f51\u5361\u7684\u865a\u673a\u6765\u9a8c\u8bc1 OVS-DPDK \u529f\u80fd\u3002</p> <p>\u5b89\u88c5 KVM Device Plugin \u6765\u521b\u5efa\u865a\u673a\uff0c\u66f4\u591a\u4f7f\u7528\u65b9\u5f0f\u8bf7\u53c2\u8003 KVM Device Plugin\u3002</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/kubevirt/kubernetes-device-plugins/master/manifests/kvm-ds.yml\n</code></pre> <p>\u521b\u5efa NetworkAttachmentDefinition\uff1a</p> <pre><code>apiVersion: k8s.cni.cncf.io/v1\nkind: NetworkAttachmentDefinition\nmetadata:\nname: ovn-dpdk\nnamespace: default\nspec:\nconfig: &gt;-\n{\n\"cniVersion\": \"0.3.0\", \n\"type\": \"kube-ovn\", \n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\", \n\"provider\": \"ovn-dpdk.default.ovn\",\n\"vhost_user_socket_volume_name\": \"vhostuser-sockets\",\n\"vhost_user_socket_name\": \"sock\"\n}\n</code></pre> <p>\u4f7f\u7528\u4e0b\u9762\u7684 Dockerfile \u521b\u5efa VM \u955c\u50cf\uff1a</p> <pre><code>FROM quay.io/kubevirt/virt-launcher:v0.46.1\n\n# wget http://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud.qcow2\nCOPY CentOS-7-x86_64-GenericCloud.qcow2 /var/lib/libvirt/images/CentOS-7-x86_64-GenericCloud.qcow2\n</code></pre> <p>\u521b\u5efa\u865a\u62df\u673a\uff1a</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: vm-config\ndata:\nstart.sh: |\nchmod u+w /etc/libvirt/qemu.conf\necho \"hugetlbfs_mount = \\\"/dev/hugepages\\\"\" &gt;&gt; /etc/libvirt/qemu.conf\nvirtlogd &amp;\nlibvirtd &amp;\n\nmkdir /var/lock\n\nsleep 5\n\nvirsh define /root/vm/vm.xml\nvirsh start vm\n\ntail -f /dev/null\nvm.xml: |\n&lt;domain type='kvm'&gt;\n&lt;name&gt;vm&lt;/name&gt;\n&lt;uuid&gt;4a9b3f53-fa2a-47f3-a757-dd87720d9d1d&lt;/uuid&gt;\n&lt;memory unit='KiB'&gt;2097152&lt;/memory&gt;\n&lt;currentMemory unit='KiB'&gt;2097152&lt;/currentMemory&gt;\n&lt;memoryBacking&gt;\n&lt;hugepages&gt;\n&lt;page size='2' unit='M' nodeset='0'/&gt;\n&lt;/hugepages&gt;\n&lt;/memoryBacking&gt;\n&lt;vcpu placement='static'&gt;2&lt;/vcpu&gt;\n&lt;cputune&gt;\n&lt;shares&gt;4096&lt;/shares&gt;\n&lt;vcpupin vcpu='0' cpuset='4'/&gt;\n&lt;vcpupin vcpu='1' cpuset='5'/&gt;\n&lt;emulatorpin cpuset='1,3'/&gt;\n&lt;/cputune&gt;\n&lt;os&gt;\n&lt;type arch='x86_64' machine='pc'&gt;hvm&lt;/type&gt;\n&lt;boot dev='hd'/&gt;\n&lt;/os&gt;\n&lt;features&gt;\n&lt;acpi/&gt;\n&lt;apic/&gt;\n&lt;/features&gt;\n&lt;cpu mode='host-model'&gt;\n&lt;model fallback='allow'/&gt;\n&lt;topology sockets='1' cores='2' threads='1'/&gt;\n&lt;numa&gt;\n&lt;cell id='0' cpus='0-1' memory='2097152' unit='KiB' memAccess='shared'/&gt;\n&lt;/numa&gt;\n&lt;/cpu&gt;\n&lt;on_reboot&gt;restart&lt;/on_reboot&gt;\n&lt;devices&gt;\n&lt;emulator&gt;/usr/libexec/qemu-kvm&lt;/emulator&gt;\n&lt;disk type='file' device='disk'&gt;\n&lt;driver name='qemu' type='qcow2' cache='none'/&gt;\n&lt;source file='/var/lib/libvirt/images/CentOS-7-x86_64-GenericCloud.qcow2'/&gt;\n&lt;target dev='vda' bus='virtio'/&gt;\n&lt;/disk&gt;\n\n&lt;interface type='vhostuser'&gt;\n&lt;mac address='00:00:00:0A:30:89'/&gt;\n&lt;source type='unix' path='/var/run/vm/sock' mode='server'/&gt;\n&lt;model type='virtio'/&gt;\n&lt;driver queues='2'&gt;\n&lt;host mrg_rxbuf='off'/&gt;\n&lt;/driver&gt;\n&lt;/interface&gt;\n&lt;serial type='pty'&gt;\n&lt;target type='isa-serial' port='0'&gt;\n&lt;model name='isa-serial'/&gt;\n&lt;/target&gt;\n&lt;/serial&gt;\n&lt;console type='pty'&gt;\n&lt;target type='serial' port='0'/&gt;\n&lt;/console&gt;\n&lt;channel type='unix'&gt;\n&lt;source mode='bind' path='/var/lib/libvirt/qemu/channel/target/domain-1-vm/org.qemu.guest_agent.0'/&gt;\n&lt;target type='virtio' name='org.qemu.guest_agent.0' state='connected'/&gt;\n&lt;alias name='channel0'/&gt;\n&lt;address type='virtio-serial' controller='0' bus='0' port='1'/&gt;\n&lt;/channel&gt;\n\n&lt;/devices&gt;\n&lt;/domain&gt;\n---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\nname: vm-deployment\nlabels:\napp: vm\nspec:\nreplicas: 1\nselector:\nmatchLabels:\napp: vm\ntemplate:\nmetadata:\nlabels:\napp: vm\nannotations:\nk8s.v1.cni.cncf.io/networks: default/ovn-dpdk\novn-dpdk.default.ovn.kubernetes.io/ip_address: 10.16.0.96\novn-dpdk.default.ovn.kubernetes.io/mac_address: 00:00:00:0A:30:89\nspec:\nnodeSelector:\novn.kubernetes.io/ovs_dp_type: userspace\nsecurityContext:\nrunAsUser: 0\nvolumes:\n- name: vhostuser-sockets\nemptyDir: {}\n- name: xml\nconfigMap:\nname: vm-config\n- name: hugepage\nemptyDir:\nmedium: HugePages-2Mi\n- name: libvirt-runtime\nemptyDir: {}\ncontainers:\n- name: vm\nimage: vm-vhostuser:latest\ncommand: [\"bash\", \"/root/vm/start.sh\"]\nsecurityContext:\ncapabilities:\nadd:\n- NET_BIND_SERVICE\n- SYS_NICE\n- NET_RAW\n- NET_ADMIN\nprivileged: false\nrunAsUser: 0\nresources:\nlimits:\ncpu: '2'\ndevices.kubevirt.io/kvm: '1'\nmemory: '8784969729'\nhugepages-2Mi: 2Gi\nrequests:\ncpu: 666m\ndevices.kubevirt.io/kvm: '1'\nephemeral-storage: 50M\nmemory: '4490002433'\nvolumeMounts:\n- name: vhostuser-sockets\nmountPath: /var/run/vm\n- name: xml\nmountPath: /root/vm/\n- mountPath: /dev/hugepages\nname: hugepage\n- name: libvirt-runtime\nmountPath: /var/run/libvirt\n</code></pre> <p>\u7b49\u5f85\u865a\u62df\u673a\u521b\u5efa\u6210\u529f\u540e\u8fdb\u5165 Pod \u8fdb\u884c\u865a\u673a\u914d\u7f6e\uff1a</p> <pre><code># virsh set-user-password vm root 12345\nPassword set successfully for root in vm\n\n# virsh console vm\nConnected to domain 'vm'\nEscape character is ^] (Ctrl + ])\n\nCentOS Linux 7 (Core)\nKernel 3.10.0-1127.el7.x86_64 on an x86_64\n\nlocalhost login: root\nPassword:\nLast login: Fri Feb 25 09:52:54 on ttyS0\n</code></pre> <p>\u63a5\u4e0b\u6765\u53ef\u4ee5\u767b\u5f55\u865a\u673a\u8fdb\u884c\u7f51\u7edc\u914d\u7f6e\u5e76\u6d4b\u8bd5\uff1a</p> <pre><code>ip link set eth0 mtu 1400\nip addr add 10.16.0.96/16 dev eth0\nip ro add default via 10.16.0.1\nping 114.114.114.114\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/external-gateway/","title":"\u5916\u90e8\u7f51\u5173\u8bbe\u7f6e","text":"<p>\u5728\u4e00\u4e9b\u573a\u666f\u4e0b\uff0c\u5bf9\u6240\u6709\u5bb9\u5668\u8bbf\u95ee\u5916\u90e8\u7684\u6d41\u91cf\u9700\u8981\u901a\u8fc7\u4e00\u4e2a\u5916\u90e8\u7684\u7f51\u5173\u8fdb\u884c\u7edf\u4e00\u7684\u7ba1\u7406\u548c\u5ba1\u8ba1\u3002 Kube-OVN \u53ef\u4ee5\u901a\u8fc7\u5728\u5b50\u7f51\u4e2d\u8fdb\u884c\u76f8\u5e94\u7684\u8def\u7531\u914d\u7f6e\uff0c\u5c06\u51fa\u7f51\u6d41\u91cf\u8f6c\u53d1\u81f3\u5bf9\u5e94\u7684\u5916\u90e8\u7f51\u5173\u3002</p>"},{"location":"advance/external-gateway/#_2","title":"\u4f7f\u7528\u65b9\u5f0f","text":"<pre><code>kind: Subnet\napiVersion: kubeovn.io/v1\nmetadata:\nname: external\nspec:\ncidrBlock: 172.31.0.0/16\ngatewayType: centralized\nnatOutgoing: false\nexternalEgressGateway: 192.168.0.1\npolicyRoutingTableID: 1000\npolicyRoutingPriority: 1500\n</code></pre> <ul> <li><code>natOutgoing</code>: \u9700\u8981\u8bbe\u7f6e\u4e3a <code>false</code>\u3002</li> <li><code>externalEgressGateway</code>\uff1a\u8bbe\u7f6e\u4e3a\u5916\u90e8\u7f51\u5173\u7684\u5730\u5740\uff0c\u9700\u8981\u548c\u7f51\u5173\u8282\u70b9\u5728\u540c\u4e00\u4e2a\u4e8c\u5c42\u53ef\u8fbe\u57df\u3002</li> <li><code>policyRoutingTableID</code>\uff1a\u4f7f\u7528\u7684\u672c\u5730\u7b56\u7565\u8def\u7531\u8868\u7684 TableID \u6bcf\u4e2a\u5b50\u7f51\u5747\u9700\u4e0d\u540c\u4ee5\u907f\u514d\u51b2\u7a81\u3002</li> <li><code>policyRoutingPriority</code>\uff1a\u8def\u7531\u4f18\u5148\u7ea7\uff0c\u4e3a\u907f\u514d\u540e\u7eed\u7528\u6237\u5b9a\u5236\u5316\u7684\u5176\u4ed6\u8def\u7531\u64cd\u4f5c\u51b2\u7a81\uff0c\u8fd9\u91cc\u53ef\u4ee5\u6307\u5b9a\u8def\u7531\u4f18\u5148\u7ea7\uff0c\u82e5\u65e0\u7279\u6b8a\u9700\u6c42\u586b\u5165\u4efb\u610f\u503c\u5373\u53ef\u3002</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/fastpath/","title":"\u624b\u52a8\u7f16\u8bd1 FastPath \u6a21\u5757","text":"<p>\u7ecf\u8fc7\u6570\u636e\u5e73\u9762\u7684\u6027\u80fd Profile\uff0c<code>Netfilter</code> \u5728\u5bb9\u5668\u5185\u548c\u5bbf\u4e3b\u673a\u4e0a\u7684\u76f8\u5173\u5904\u7406\u6d88\u8017\u4e86 20% \u5de6\u53f3\u7684 CPU \u8d44\u6e90\uff0cFastPath \u6a21\u5757\u53ef\u4ee5\u7ed5\u8fc7 <code>Netfilter</code> \u4ece\u800c \u964d\u4f4e CPU \u7684\u6d88\u8017\u548c\u5ef6\u8fdf\uff0c\u5e76\u63d0\u5347\u541e\u5410\u91cf\u3002\u672c\u6587\u6863\u5c06\u4ecb\u7ecd\u5982\u4f55\u624b\u52a8\u7f16\u8bd1 FastPath \u6a21\u5757\u3002</p>"},{"location":"advance/fastpath/#_1","title":"\u4e0b\u8f7d\u76f8\u5173\u5185\u6838\u6a21\u5757\u4ee3\u7801","text":"<pre><code>git clone --depth=1 https://github.com/kubeovn/kube-ovn.git\n</code></pre>"},{"location":"advance/fastpath/#_2","title":"\u5b89\u88c5\u4f9d\u8d56","text":"<p>\u8fd9\u91cc\u4ee5 CentOS \u4e3a\u4f8b\u4e0b\u8f7d\u76f8\u5173\u4f9d\u8d56\uff1a</p> <pre><code>yum install -y kernel-devel-$(uname -r) gcc elfutils-libelf-devel\n</code></pre>"},{"location":"advance/fastpath/#_3","title":"\u7f16\u8bd1\u76f8\u5173\u6a21\u5757","text":"<p>\u9488\u5bf9 3.x \u7684\u5185\u6838\uff1a</p> <pre><code>cd kube-ovn/fastpath\nmake all\n</code></pre> <p>\u9488\u5bf9 4.x \u7684\u5185\u6838\uff1a</p> <pre><code>cd kube-ovn/fastpath/4.18\ncp ../Makefile .\nmake all\n</code></pre>"},{"location":"advance/fastpath/#_4","title":"\u5b89\u88c5\u5185\u6838\u6a21\u5757","text":"<p>\u5c06 <code>kube_ovn_fastpath.ko</code> \u590d\u5236\u5230\u6bcf\u4e2a\u9700\u8981\u6027\u80fd\u4f18\u5316\u7684\u8282\u70b9 <code>/tmp</code> \u76ee\u5f55\u4e0b\uff0c<code>kube-ovn-cni</code> \u4f1a\u81ea\u52a8\u52a0\u8f7d\u8be5\u6a21\u5757\u3002</p> <p>\u4f7f\u7528 <code>dmesg</code> \u786e\u8ba4\u5b89\u88c5\u6210\u529f\uff1a</p> <pre><code># dmesg\n[619631.323788] init_module,kube_ovn_fastpath_local_out\n[619631.323798] init_module,kube_ovn_fastpath_post_routing\n[619631.323800] init_module,kube_ovn_fastpath_pre_routing\n[619631.323801] init_module,kube_ovn_fastpath_local_in\n</code></pre> <p>\u5982\u9700\u5378\u8f7d\u6a21\u5757\uff0c\u53ef\u5c06\u8be5\u6a21\u5757\u4ece <code>/tmp</code> \u76ee\u5f55\u4e0b\u79fb\u9664\uff0c<code>kube-ovn-cni</code> \u4f1a\u81ea\u52a8\u5378\u8f7d\u8be5\u6a21\u5757\u3002</p> <p>\u8be5\u6a21\u5757\u5728\u673a\u5668\u91cd\u542f\u540e\u4e0d\u4f1a\u81ea\u52a8\u52a0\u8f7d\uff0c\u5982\u9700\u81ea\u52a8\u52a0\u8f7d\u8bf7\u6839\u636e\u7cfb\u7edf\u5f04\u914d\u7f6e\u7f16\u5199\u76f8\u5e94\u81ea\u542f\u52a8\u811a\u672c\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/multi-nic/","title":"\u591a\u7f51\u5361\u7ba1\u7406","text":"<p>Kube-OVN \u53ef\u4ee5\u4e3a\u5176\u4ed6 CNI \u7f51\u7edc\u63d2\u4ef6\uff0c\u4f8b\u5982 macvlan\u3001vlan\u3001host-device \u7b49\u63d2\u4ef6\u63d0\u4f9b\u96c6\u7fa4\u7ea7\u522b\u7684 IPAM \u80fd\u529b\uff0c \u5176\u4ed6\u7f51\u7edc\u63d2\u4ef6\u4e5f\u53ef\u4ee5\u4f7f\u7528\u5230 Kube-OVN \u4e2d\u5b50\u7f51\u4ee5\u53ca\u56fa\u5b9a IP \u529f\u80fd\u3002</p> <p>\u540c\u65f6 Kube-OVN \u4e5f\u652f\u6301\u591a\u5757\u7f51\u5361\u5747\u4e3a Kube-OVN \u7c7b\u578b\u7f51\u5361\u60c5\u51b5\u4e0b\u7684\u5730\u5740\u7ba1\u7406\u3002</p>"},{"location":"advance/multi-nic/#_2","title":"\u5de5\u4f5c\u539f\u7406","text":"<p>\u901a\u8fc7\u4f7f\u7528 Multus CNI, \u6211\u4eec\u53ef\u4ee5\u7ed9\u4e00\u4e2a Pod \u6dfb\u52a0\u591a\u5757\u4e0d\u540c\u7f51\u7edc\u7684\u7f51\u5361\u3002 \u7136\u800c\u6211\u4eec\u4ecd\u7136\u7f3a\u4e4f\u5bf9\u96c6\u7fa4\u8303\u56f4\u5185\u4e0d\u540c\u7f51\u7edc\u7684 IP \u5730\u5740\u8fdb\u884c\u7ba1\u7406\u7684\u80fd\u529b\u3002\u5728 Kube-OVN \u4e2d\uff0c\u6211\u4eec\u5df2\u7ecf\u80fd\u591f\u901a\u8fc7 Subnet \u548c IP \u7684 CRD \u6765\u8fdb\u884c IP \u7684\u9ad8\u7ea7\u7ba1\u7406\uff0c \u4f8b\u5982\u5b50\u7f51\u7ba1\u7406\uff0cIP \u9884\u7559\uff0c\u968f\u673a\u5206\u914d\uff0c\u56fa\u5b9a\u5206\u914d\u7b49\u3002\u73b0\u5728\u6211\u4eec\u5bf9\u5b50\u7f51\u8fdb\u884c\u6269\u5c55\uff0c\u6765\u63a5\u5165\u5176\u4ed6\u4e0d\u540c\u7684\u7f51\u7edc\u63d2\u4ef6\uff0c\u4f7f\u5f97\u5176\u4ed6\u7f51\u7edc\u63d2\u4ef6\u4e5f\u53ef\u4ee5\u4f7f\u7528 Kube-OVN \u7684 IPAM \u529f\u80fd\u3002</p>"},{"location":"advance/multi-nic/#_3","title":"\u5de5\u4f5c\u6d41\u7a0b","text":"<p>\u4e0a\u56fe\u5c55\u793a\u4e86\u5982\u4f55\u901a\u8fc7 Kube-OVN \u6765\u7ba1\u7406\u5176\u4ed6\u7f51\u7edc\u63d2\u4ef6\u7684 IP \u5730\u5740\u3002\u5176\u4e2d\u5bb9\u5668\u7684 eth0 \u7f51\u5361\u63a5\u5165 OVN \u7f51\u7edc\uff0cnet1 \u7f51\u5361\u63a5\u5165\u5176\u4ed6 CNI \u7f51\u7edc\u3002 net1 \u7f51\u7edc\u7684\u7f51\u7edc\u5b9a\u4e49\u6765\u81ea\u4e8e multus-cni \u4e2d\u7684 NetworkAttachmentDefinition \u8d44\u6e90\u5b9a\u4e49\u3002</p> <p>\u5f53 Pod \u521b\u5efa\u65f6\uff0c<code>kube-ovn-controller</code> \u4f1a\u76d1\u542c\u5230 Pod \u6dfb\u52a0\u4e8b\u4ef6\uff0c\u5e76\u6839\u636e Pod \u4e2d\u7684 annotation \u53bb\u5bfb\u627e\u5230\u5bf9\u5e94\u7684 Subnet \u5e76\u4ece\u4e2d\u8fdb\u884c IP \u7684\u5206\u914d\u548c\u7ba1\u7406\uff0c \u5e76\u5c06 Pod \u6240\u5206\u914d\u5230\u7684\u5730\u5740\u4fe1\u606f\u5199\u56de\u5230 Pod annotation \u4e2d\u3002</p> <p>\u5728\u5bb9\u5668\u6240\u5728\u673a\u5668\u7684 CNI \u53ef\u4ee5\u901a\u8fc7\u5728\u914d\u7f6e\u4e2d\u914d\u7f6e <code>kube-ovn-cni</code> \u4f5c\u4e3a ipam \u63d2\u4ef6, <code>kube-ovn-cni</code> \u5c06\u4f1a\u8bfb\u53d6 Pod annotation \u5e76\u5c06\u5730\u5740\u4fe1\u606f\u901a\u8fc7 CNI \u534f\u8bae\u7684\u6807\u51c6\u683c\u5f0f\u8fd4\u56de\u7ed9\u76f8\u5e94\u7684 CNI \u63d2\u4ef6\u3002</p>"},{"location":"advance/multi-nic/#_4","title":"\u4f7f\u7528\u65b9\u6cd5","text":""},{"location":"advance/multi-nic/#kube-ovn-multus","title":"\u5b89\u88c5 Kube-OVN \u548c Multus","text":"<p>\u8bf7\u53c2\u8003 Kube-OVN \u4e00\u952e\u5b89\u88c5 \u548c Multus how to use \u6765\u5b89\u88c5 Kube-OVN \u548c Multus-CNI\u3002</p>"},{"location":"advance/multi-nic/#cni-ipam","title":"\u4e3a\u5176\u4ed6 CNI \u63d0\u4f9b IPAM","text":"<p>\u6b64\u65f6\u4e3b\u7f51\u5361\u4e3a Kube-OVN \u7c7b\u578b\u7f51\u5361\uff0c\u9644\u5c5e\u7f51\u5361\u4e3a\u5176\u4ed6\u7c7b\u578b CNI\u3002</p>"},{"location":"advance/multi-nic/#networkattachmentdefinition","title":"\u521b\u5efa NetworkAttachmentDefinition","text":"<p>\u8fd9\u91cc\u6211\u4eec\u4f7f\u7528 macvlan \u4f5c\u4e3a\u5bb9\u5668\u7f51\u7edc\u7684\u7b2c\u4e8c\u4e2a\u7f51\u7edc\uff0c\u5e76\u5c06\u5176 ipam \u8bbe\u7f6e\u4e3a <code>kube-ovn</code>\uff1a</p> <pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: macvlan\nnamespace: default\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.0\",\n\"type\": \"macvlan\",\n\"master\": \"eth0\",\n\"mode\": \"bridge\",\n\"ipam\": {\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"macvlan.default\"\n}\n}'\n</code></pre> <ul> <li><code>spec.config.ipam.type</code>: \u9700\u8981\u4e3a <code>kube-ovn</code> \u6765\u8c03\u7528 kube-ovn \u7684\u63d2\u4ef6\u6765\u83b7\u53d6\u5730\u5740\u4fe1\u606f\u3002</li> <li><code>server_socket</code>: Kube-OVN \u901a\u4fe1\u4f7f\u7528\u7684 socket \u6587\u4ef6\u3002 \u9ed8\u8ba4\u4f4d\u7f6e\u4e3a <code>/run/openvswitch/kube-ovn-daemon.sock</code>\u3002</li> <li><code>provider</code>: \u5f53\u524d NetworkAttachmentDefinition \u7684 <code>&lt;name&gt;.&lt;namespace&gt;</code> , Kube-OVN \u5c06\u4f1a\u4f7f\u7528\u8fd9\u4e9b\u4fe1\u606f\u627e\u5230\u5bf9\u5e94\u7684 Subnet \u8d44\u6e90\u3002</li> </ul>"},{"location":"advance/multi-nic/#kube-ovn","title":"\u9644\u5c5e\u7f51\u5361\u4e3a Kube-OVN \u7c7b\u578b\u7f51\u5361","text":"<p>\u6b64\u65f6\u591a\u5757\u7f51\u5361\u5747\u4e3a Kube-OVN \u7c7b\u578b\u7f51\u5361\u3002</p>"},{"location":"advance/multi-nic/#networkattachmentdefinition_1","title":"\u521b\u5efa NetworkAttachmentDefinition","text":"<p>\u5c06 <code>provider</code> \u7684\u540e\u7f00\u8bbe\u7f6e\u4e3a <code>ovn</code>\uff1a</p> <pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: attachnet\nnamespace: default\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.0\",\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"attachnet.default.ovn\"\n}'\n</code></pre> <ul> <li><code>spec.config.type</code>: \u8bbe\u7f6e\u4e3a <code>kube-ovn</code> \u6765\u89e6\u53d1 CNI \u63d2\u4ef6\u4f7f\u7528 Kube-OVN \u5b50\u7f51\u3002</li> <li><code>server_socket</code>: Kube-OVN \u901a\u4fe1\u4f7f\u7528\u7684 socket \u6587\u4ef6\u3002 \u9ed8\u8ba4\u4f4d\u7f6e\u4e3a <code>/run/openvswitch/kube-ovn-daemon.sock</code>\u3002</li> <li><code>provider</code>: \u5f53\u524d NetworkAttachmentDefinition \u7684 <code>&lt;name&gt;.&lt;namespace&gt;.ovn</code> , Kube-OVN \u5c06\u4f1a\u4f7f\u7528\u8fd9\u4e9b\u4fe1\u606f\u627e\u5230\u5bf9\u5e94\u7684 Subnet \u8d44\u6e90\uff0c\u6ce8\u610f\u540e\u7f00\u9700\u8981\u8bbe\u7f6e\u4e3a ovn\u3002</li> </ul>"},{"location":"advance/multi-nic/#kube-ovn-subnet","title":"\u521b\u5efa\u4e00\u4e2a Kube-OVN Subnet","text":"<p>\u521b\u5efa\u4e00\u4e2a Kube-OVN Subnet,\u8bbe\u7f6e\u5bf9\u5e94\u7684 <code>cidrBlock</code> \u548c <code>exclude_ips</code>, <code>provider</code> \u5e94\u8be5\u8bbe\u7f6e\u4e3a\u5bf9\u5e94\u7684 NetworkAttachmentDefinition \u7684 <code>&lt;name&gt;.&lt;namespace&gt;</code>, \u4f8b\u5982\u7528 macvlan \u63d0\u4f9b\u9644\u52a0\u7f51\u5361\uff0c\u521b\u5efa Subnet \u5982\u4e0b\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: macvlan\nspec:\nprotocol: IPv4\nprovider: macvlan.default\ncidrBlock: 172.17.0.0/16\ngateway: 172.17.0.1\nexcludeIps:\n- 172.17.0.0..172.17.0.10\n</code></pre> <p><code>gateway</code>, <code>private</code>, <code>nat</code> \u53ea\u5bf9 <code>provider</code> \u7c7b\u578b\u4e3a ovn \u7684\u7f51\u7edc\u751f\u6548\uff0c\u4e0d\u9002\u7528\u4e8e attachment network\u3002</p> <p>\u5982\u679c\u4ee5 Kube-OVN \u4f5c\u4e3a\u9644\u52a0\u7f51\u5361\uff0c\u5219 <code>provider</code> \u5e94\u8be5\u8bbe\u7f6e\u4e3a\u5bf9\u5e94\u7684 NetworkAttachmentDefinition \u7684 <code>&lt;name&gt;.&lt;namespace&gt;.ovn</code>\uff0c\u5e76\u8981\u4ee5 <code>ovn</code> \u4f5c\u4e3a\u540e\u7f00\u7ed3\u675f\u3002 \u7528 Kube-OVN \u63d0\u4f9b\u9644\u52a0\u7f51\u5361\uff0c\u521b\u5efa Subnet \u793a\u4f8b\u5982\u4e0b\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: attachnet\nspec:\nprotocol: IPv4\nprovider: attachnet.default.ovn\ncidrBlock: 172.17.0.0/16\ngateway: 172.17.0.1\nexcludeIps:\n- 172.17.0.0..172.17.0.10\n</code></pre>"},{"location":"advance/multi-nic/#pod","title":"\u521b\u5efa\u4e00\u4e2a\u591a\u7f51\u7edc\u7684 Pod","text":"<p>\u5bf9\u4e8e\u5730\u5740\u968f\u673a\u5206\u914d\u7684 Pod\uff0c\u53ea\u9700\u8981\u6dfb\u52a0\u5982\u4e0b annotation <code>k8s.v1.cni.cncf.io/networks</code>,\u53d6\u503c\u4e3a\u5bf9\u5e94\u7684 NetworkAttachmentDefinition \u7684 <code>&lt;namespace&gt;/&lt;name&gt;</code>\uff1a</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: samplepod\nnamespace: default\nannotations:\nk8s.v1.cni.cncf.io/networks: default/macvlan\nspec:\ncontainers:\n- name: samplepod\ncommand: [\"/bin/ash\", \"-c\", \"trap : TERM INT; sleep infinity &amp; wait\"]\nimage: docker.io/library/alpine:edge\n</code></pre>"},{"location":"advance/multi-nic/#ip-pod","title":"\u521b\u5efa\u56fa\u5b9a IP \u7684 Pod","text":"<p>\u5bf9\u4e8e\u56fa\u5b9a IP \u7684 Pod\uff0c\u6dfb\u52a0 <code>&lt;networkAttachmentName&gt;.&lt;networkAttachmentNamespace&gt;.kubernetes.io/ip_address</code> annotation\uff1a</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: static-ip\nnamespace: default\nannotations:\nk8s.v1.cni.cncf.io/networks: default/macvlan\novn.kubernetes.io/ip_address: 10.16.0.15\novn.kubernetes.io/mac_address: 00:00:00:53:6B:B6\nmacvlan.default.kubernetes.io/ip_address: 172.17.0.100\nmacvlan.default.kubernetes.io/mac_address: 00:00:00:53:6B:BB\nspec:\ncontainers:\n- name: static-ip\nimage: docker.io/library/nginx:alpine\n</code></pre>"},{"location":"advance/multi-nic/#ip","title":"\u521b\u5efa\u4f7f\u7528\u56fa\u5b9a IP \u7684\u5de5\u4f5c\u8d1f\u8f7d","text":"<p>\u5bf9\u4e8e\u4f7f\u7528 ippool \u7684\u5de5\u4f5c\u8d1f\u8f7d, \u6dfb\u52a0 <code>&lt;networkAttachmentName&gt;.&lt;networkAttachmentNamespace&gt;.kubernetes.io/ip_pool</code> annotations:</p> <pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nnamespace: default\nname: static-workload\nlabels:\napp: static-workload\nspec:\nreplicas: 2\nselector:\nmatchLabels:\napp: static-workload\ntemplate:\nmetadata:\nlabels:\napp: static-workload\nannotations:\nk8s.v1.cni.cncf.io/networks: default/macvlan\novn.kubernetes.io/ip_pool: 10.16.0.15,10.16.0.16,10.16.0.17\nmacvlan.default.kubernetes.io/ip_pool: 172.17.0.200,172.17.0.201,172.17.0.202\nspec:\ncontainers:\n- name: static-workload\nimage: docker.io/library/nginx:alpine\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/nat-policy-rule/","title":"\u9ed8\u8ba4 VPC NAT \u7b56\u7565\u89c4\u5219","text":""},{"location":"advance/nat-policy-rule/#_1","title":"\u7528\u9014","text":"<p>\u9ed8\u8ba4 VPC \u4e0b\u7684 Overlay \u5b50\u7f51\uff0c\u6253\u5f00 <code>natOutgoing</code> \u5f00\u5173\u65f6\uff0cSubnet \u4e0b\u7684\u6240\u6709 Pod \u8bbf\u95ee\u5916\u7f51\u90fd\u9700\u8981\u505a SNAT \u6210\u8282\u70b9\u7684 IP\uff0c\u4f46\u662f\u6709\u4e9b\u573a\u666f\u6211\u4eec\u5e76\u4e0d\u5e0c\u671b\u5b50\u7f51\u5185\u6240\u6709 Pod \u8bbf\u95ee\u5916\u7f51\u90fd\u505a SNAT\u3002</p> <p>\u56e0\u6b64 NAT \u7b56\u7565\u5c31\u662f\u4e3a\u4e86\u63d0\u4f9b\u4e00\u4e2a\u63a5\u53e3\u8ba9\u7528\u6237\u51b3\u5b9a\u5b50\u7f51\u5185\u7684\u54ea\u4e9b CIDR \u6216\u8005 IP \u8bbf\u95ee\u5916\u7f51\u505a SNAT\u3002</p>"},{"location":"advance/nat-policy-rule/#_2","title":"\u4f7f\u7528\u65b9\u6cd5","text":"<p>\u5728 <code>subnet.Spec</code> \u4e2d\u5f00\u542f <code>natOutgoing</code>\u5f00\u5173\uff0c \u5e76\u4e14\u6dfb\u52a0\u5b57\u6bb5 <code>natOutgoingPolicyRules</code> \u5982\u4e0b\uff1a</p> <pre><code>spec:\nnatOutgoing: true\nnatOutgoingPolicyRules:\n- action: forward\nmatch:\nsrcIPs: 10.0.11.0/30,10.0.11.254\n- action: nat\nmatch:\nsrcIPs: 10.0.11.128/26\ndstIPs: 114.114.114.114,8.8.8.8\n</code></pre> <p>\u4ee5\u4e0a\u6848\u4f8b\u8868\u793a\u6709\u4e24\u6761 NAT \u7b56\u7565\u89c4\u5219\uff1a</p> <ol> <li>\u6e90 IP \u662f 10.0.11.0/30 \u6216\u8005 10.0.11.254  \u7684\u62a5\u6587\u8bbf\u95ee\u5916\u7f51\u65f6\u4e0d\u4f1a\u505a SNAT\u3002</li> <li>\u6e90 IP \u662f 10.0.11.128/26 \u5e76\u4e14\u76ee\u7684 IP \u662f 114.114.114.114 \u6216\u8005 8.8.8.8 \u7684\u62a5\u6587\u8bbf\u95ee\u5916\u7f51\u65f6\u4f1a\u505a SNAT\u3002</li> </ol> <p>\u5b57\u6bb5\u63cf\u8ff0\uff1a</p> <p><code>action</code>\uff1a\u6ee1\u8db3 <code>match</code> \u5bf9\u5e94\u6761\u4ef6\u7684\u62a5\u6587\uff0c\u4f1a\u6267\u884c\u7684 action, action \u5206\u4e3a\u4e24\u79cd <code>forward</code> \u548c <code>nat</code> \uff0c<code>forward</code> \u8868\u793a\u62a5\u6587\u51fa\u5916\u7f51\u4e0d\u505a SNAT, <code>nat</code> \u8868\u793a\u62a5\u6587\u51fa\u5916\u7f51\u505a SNAT\u3002 \u6ca1\u6709\u914d\u7f6e natOutgoingPolicyRules \u65f6\uff0c\u9ed8\u8ba4\u60c5\u51b5\u62a5\u6587\u4ecd\u7136\u662f\u505a SNAT\u3002</p> <p><code>match</code>\uff1a\u8868\u793a\u62a5\u6587\u7684\u5339\u914d\u6bb5\uff0c\u5339\u914d\u6bb5\u6709 <code>srcIPs</code> \u548c <code>dstIPs</code>\uff0c \u8fd9\u91cc\u8868\u793a\u4ece\u5b50\u7f51\u5185\u5230\u5916\u7f51\u65b9\u5411\u4e0a\u7684\u62a5\u6587\u7684\u6e90 IP \u548c \u76ee\u7684 IP\u3002<code>match.srcIPs</code> \u548c <code>match.dstIPs</code> \u652f\u6301\u591a\u4e2a CIDR \u548c IP\uff0c\u4e4b\u95f4\u7528\u9017\u53f7\u95f4\u9694\u3002</p> <p>\u5982\u679c\u51fa\u73b0\u591a\u4e2a match \u89c4\u5219\u91cd\u53e0\uff0c\u5219\u6309\u7167 <code>natOutgoingPolicyRules</code> \u6570\u7ec4\u987a\u5e8f\u8fdb\u884c\u5339\u914d\uff0c\u6700\u5148\u88ab\u5339\u914d\u7684 action \u4f1a\u88ab\u6267\u884c\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/node-local-dns/","title":"\u8282\u70b9\u672c\u5730 DNS \u7f13\u5b58\u548c Kube-OVN \u9002\u914d","text":"<p>NodeLocal DNSCache \u662f\u901a\u8fc7\u96c6\u7fa4\u8282\u70b9\u4e0a\u4f5c\u4e3a DaemonSet \u8fd0\u884c DNS \u7f13\u5b58\u6765\u63d0\u9ad8\u96c6\u7fa4 DNS \u6027\u80fd\uff0c\u8be5\u529f\u80fd\u4e5f\u53ef\u4ee5\u548c Kube-OVN \u9002\u914d\u3002</p>"},{"location":"advance/node-local-dns/#dns","title":"\u8282\u70b9\u672c\u5730 DNS \u7f13\u5b58\u90e8\u7f72","text":""},{"location":"advance/node-local-dns/#kubernetes-dns","title":"\u90e8\u7f72 Kubernetes \u7684\u8282\u70b9\u672c\u5730 DNS \u7f13\u5b58","text":"<p>\u8be5\u6b65\u9aa4\u53c2\u8003 Kubernetes \u5b98\u7f51\u914d\u7f6e Nodelocaldnscache\u3002</p> <p>\u4f7f\u7528\u4ee5\u4e0b\u811a\u672c\u90e8\u7f72\uff1a</p> <pre><code>#!bin/bash\n\nlocaldns=169.254.20.10\ndomain=cluster.local\nkubedns=10.96.0.10\n\nwget https://raw.githubusercontent.com/kubernetes/kubernetes/master/cluster/addons/dns/nodelocaldns/nodelocaldns.yaml\nsed -i \"s/__PILLAR__LOCAL__DNS__/$localdns/g; s/__PILLAR__DNS__DOMAIN__/$domain/g; s/,__PILLAR__DNS__SERVER__//g; s/__PILLAR__CLUSTER__DNS__/$kubedns/g\" nodelocaldns.yaml\n\nkubectl apply -f nodelocaldns.yaml\n</code></pre> <p>\u4fee\u6539\u6bcf\u4e2a\u8282\u70b9\u4e0a\u7684 kubelet \u914d\u7f6e\u6587\u4ef6\uff0c\u5c06 <code>/var/lib/kubelet/config.yaml</code> \u4e2d\u7684 clusterDNS \u5b57\u6bb5\u4fee\u6539\u4e3a\u672c\u5730 DNS IP 169.254.20.10\uff0c\u7136\u540e\u91cd\u542f kubelet \u670d\u52a1\u3002</p>"},{"location":"advance/node-local-dns/#kube-ovn-dns","title":"Kube-OVN \u76f8\u5e94 DNS \u914d\u7f6e","text":"<p>\u90e8\u7f72\u597d Kubernetes \u7684 Nodelocal DNScache \u7ec4\u4ef6\u540e\uff0c Kube-OVN \u9700\u8981\u505a\u51fa\u4e0b\u9762\u4fee\u6539\uff1a</p>"},{"location":"advance/node-local-dns/#underlay-subnet-u2o","title":"Underlay Subnet \u5f00\u542f U2O \u5f00\u5173","text":"<p>\u5982\u679c\u662f Underlay Subnet \u9700\u8981\u4f7f\u7528\u672c\u5730 DNS \u529f\u80fd\uff0c\u9700\u8981\u5f00\u542f U2O \u529f\u80fd\uff0c\u5373\u5728 <code>kubectl edit subnet {your subnet}</code> \u4e2d\u914d\u7f6e <code>spec.u2oInterconnection = true</code> , \u5982\u679c\u662f Overlay Subnet \u5219\u4e0d\u9700\u8981\u8fd9\u6b65\u64cd\u4f5c\u3002</p>"},{"location":"advance/node-local-dns/#kube-ovn-controller-dns-ip","title":"\u7ed9 Kube-ovn-controller \u6307\u5b9a\u5bf9\u5e94\u7684\u672c\u5730 DNS IP","text":"<pre><code>kubectl edit deployment kube-ovn-controller -n kube-system\n</code></pre> <p>\u7ed9 <code>spec.template.spec.containers.args</code> \u589e\u52a0\u5b57\u6bb5 <code>--node-local-dns-ip=169.254.20.10</code></p>"},{"location":"advance/node-local-dns/#pod","title":"\u91cd\u5efa\u5df2\u7ecf\u521b\u5efa\u7684 Pod","text":"<p>\u8fd9\u6b65\u539f\u56e0\u662f\u8ba9 Pod \u91cd\u65b0\u751f\u6210 <code>/etc/resolv.conf</code> \u8ba9 nameserver \u6307\u5411\u672c\u5730 DNS IP\uff0c\u5982\u679c\u6ca1\u6709\u91cd\u5efa Pod \u7684 nameserver \u5c06\u4ecd\u7136\u4f7f\u7528\u96c6\u7fa4\u7684 DNS ClusterIP\u3002\u540c\u65f6 u2o \u5f00\u5173\u5982\u679c\u5f00\u542f\u4e5f\u9700\u8981\u91cd\u5efa Pod \u6765\u91cd\u65b0\u751f\u6210 Pod \u7f51\u5173\u3002</p>"},{"location":"advance/node-local-dns/#dns_1","title":"\u9a8c\u8bc1\u8282\u70b9\u672c\u5730 DNS \u7f13\u5b58\u529f\u80fd","text":"<p>\u4ee5\u4e0a\u914d\u7f6e\u5b8c\u6210\u540e\u53ef\u4ee5\u627e\u5230 Pod \u9a8c\u8bc1\u5982\u4e0b\uff0c\u53ef\u4ee5\u770b\u5230 Pod \u7684 DNS \u670d\u52a1\u5668\u662f\u6307\u5411\u672c\u5730 169.254.20.10 \uff0c\u5e76\u6210\u529f\u89e3\u6790\u57df\u540d\uff1a</p> <pre><code># kubectl exec -it pod1 -- nslookup github.com\nServer:         169.254.20.10\nAddress:        169.254.20.10:53\n\n\nName:   github.com\nAddress: 20.205.243.166\n</code></pre> <p>\u4e5f\u53ef\u4ee5\u5728\u8282\u70b9\u6293\u5305\u9a8c\u8bc1\u5982\u4e0b\uff0c\u53ef\u4ee5\u770b\u5230 DNS \u67e5\u8be2\u62a5\u6587\u901a\u8fc7 ovn0 \u7f51\u5361\u5230\u8fbe\u672c\u5730\u7684 DNS \u670d\u52a1\uff0cDNS \u54cd\u5e94\u62a5\u6587\u539f\u8def\u8fd4\u56de:</p> <pre><code># tcpdump -i any port 53\n\n06:20:00.441889 659246098c56_h P   ifindex 17 00:00:00:73:f1:06 ethertype IPv4 (0x0800), length 75: 10.16.0.2.40230 &gt; 169.254.20.10.53: 1291+ A? baidu.com. (27)\n06:20:00.441889 ovn0  In  ifindex 7 00:00:00:50:32:cd ethertype IPv4 (0x0800), length 75: 10.16.0.2.40230 &gt; 169.254.20.10.53: 1291+ A? baidu.com. (27)\n06:20:00.441950 659246098c56_h P   ifindex 17 00:00:00:73:f1:06 ethertype IPv4 (0x0800), length 75: 10.16.0.2.40230 &gt; 169.254.20.10.53: 1611+ AAAA? baidu.com. (27)\n06:20:00.441950 ovn0  In  ifindex 7 00:00:00:50:32:cd ethertype IPv4 (0x0800), length 75: 10.16.0.2.40230 &gt; 169.254.20.10.53: 1611+ AAAA? baidu.com. (27)\n06:20:00.442203 ovn0  Out ifindex 7 00:00:00:52:99:d8 ethertype IPv4 (0x0800), length 145: 169.254.20.10.53 &gt; 10.16.0.2.40230: 1611* 0/1/0 (97)\n06:20:00.442219 659246098c56_h Out ifindex 17 00:00:00:ea:b3:5e ethertype IPv4 (0x0800), length 145: 169.254.20.10.53 &gt; 10.16.0.2.40230: 1611* 0/1/0 (97)\n06:20:00.442273 ovn0  Out ifindex 7 00:00:00:52:99:d8 ethertype IPv4 (0x0800), length 125: 169.254.20.10.53 &gt; 10.16.0.2.40230: 1291* 2/0/0 A 39.156.66.10, A 110.242.68.66 (77)\n06:20:00.442278 659246098c56_h Out ifindex 17 00:00:00:ea:b3:5e ethertype IPv4 (0x0800), length 125: 169.254.20.10.53 &gt; 10.16.0.2.40230: 1291* 2/0/0 A 39.156.66.10, A 110.242.68.66 (77)\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/offload-corigine/","title":"\u82af\u542f\u6e90\u7f51\u5361 Offload \u652f\u6301","text":"<p>Kube-OVN \u5728\u6700\u7ec8\u7684\u6570\u636e\u5e73\u9762\u4f7f\u7528 OVS \u6765\u5b8c\u6210\u6d41\u91cf\u8f6c\u53d1\uff0c\u76f8\u5173\u7684\u6d41\u8868\u5339\u914d\uff0c\u96a7\u9053\u5c01\u88c5\u7b49\u529f\u80fd\u4e3a CPU \u5bc6\u96c6\u578b\uff0c\u5728\u5927\u6d41\u91cf\u4e0b\u4f1a\u6d88\u8017\u5927\u91cf CPU \u8d44\u6e90\u5e76\u5bfc\u81f4 \u5ef6\u8fdf\u4e0a\u5347\u548c\u541e\u5410\u91cf\u4e0b\u964d\u3002\u82af\u542f\u6e90\u7684 Agilio CX \u7cfb\u5217\u667a\u80fd\u7f51\u5361\u53ef\u4ee5\u5c06 OVS \u76f8\u5173\u7684\u64cd\u4f5c\u5378\u8f7d\u5230\u786c\u4ef6\u7f51\u5361\u4e2d\u6267\u884c\u3002 \u8be5\u6280\u672f\u53ef\u4ee5\u5728\u65e0\u9700\u5bf9 OVS \u63a7\u5236\u5e73\u9762\u8fdb\u884c\u4fee\u6539\u7684\u60c5\u51b5\u4e0b\uff0c\u7f29\u77ed\u6570\u636e\u8def\u5f84\uff0c\u907f\u514d\u5bf9\u4e3b\u673a CPU \u8d44\u6e90\u7684\u4f7f\u7528\uff0c\u5927\u5e45\u964d\u4f4e\u5ef6\u8fdf\u5e76\u663e\u8457\u63d0\u5347\u541e\u5410\u91cf\u3002</p> <p></p>"},{"location":"advance/offload-corigine/#_1","title":"\u524d\u7f6e\u6761\u4ef6","text":"<ul> <li>\u82af\u542f\u6e90 Agilio CX \u7cfb\u5217\u7684\u786c\u4ef6\u7f51\u5361\u3002</li> <li>CentOS 8 Stream \u6216\u4e0a\u6e38 Linux 5.7 \u4ee5\u4e0a\u5185\u6838\u652f\u6301\u3002</li> <li>\u7531\u4e8e\u5f53\u524d\u7f51\u5361\u4e0d\u652f\u6301 <code>dp_hash</code> \u548c <code>hash</code> \u64cd\u4f5c\u5378\u8f7d\uff0c\u9700\u5173\u95ed OVN LB \u529f\u80fd\u3002</li> </ul>"},{"location":"advance/offload-corigine/#sr-iov","title":"\u8bbe\u7f6e\u7f51\u5361 SR-IOV \u6a21\u5f0f","text":"<p>\u7528\u6237\u53ef\u53c2\u8003 Agilio Open vSwitch TC User Guide \u83b7\u5f97\u8be5\u7f51\u5361\u4f7f\u7528\u7684\u66f4\u591a\u8be6\u7ec6\u4fe1\u606f\u3002</p> <p>\u4fdd\u5b58\u4e0b\u5217\u811a\u672c\u7528\u4e8e\u540e\u7eed\u6267\u884c\u56fa\u4ef6\u76f8\u5173\u64cd\u4f5c\uff1a</p> <pre><code>#!/bin/bash\nDEVICE=${1}\nDEFAULT_ASSY=scan\nASSY=${2:-${DEFAULT_ASSY}}\nAPP=${3:-flower}\n\nif [ \"x${DEVICE}\" = \"x\" -o ! -e /sys/class/net/${DEVICE} ]; then\necho Syntax: ${0} device [ASSY] [APP]\necho\necho This script associates the TC Offload firmware\n    echo with a Netronome SmartNIC.\n    echo\necho device: is the network device associated with the SmartNIC\n    echo ASSY: defaults to ${DEFAULT_ASSY}\necho APP: defaults to flower. flower-next is supported if updated\n    echo      firmware has been installed.\n    exit 1\nfi\n\n# It is recommended that the assembly be determined by inspection\n# The following code determines the value via the debug interface\nif [ \"${ASSY}x\" = \"scanx\" ]; then\nethtool -W ${DEVICE} 0\nDEBUG=$(ethtool -w ${DEVICE} data /dev/stdout | strings)\nSERIAL=$(echo \"${DEBUG}\" | grep \"^SN:\")\nASSY=$(echo ${SERIAL} | grep -oE AMDA[0-9]{4})\nfi\n\nPCIADDR=$(basename $(readlink -e /sys/class/net/${DEVICE}/device))\nFWDIR=\"/lib/firmware/netronome\"\n\n# AMDA0081 and AMDA0097 uses the same firmware\nif [ \"${ASSY}\" = \"AMDA0081\" ]; then\nif [ ! -e ${FWDIR}/${APP}/nic_AMDA0081.nffw ]; then\nln -sf nic_AMDA0097.nffw ${FWDIR}/${APP}/nic_AMDA0081.nffw\n   fi\nfi\n\nFW=\"${FWDIR}/pci-${PCIADDR}.nffw\"\nln -sf \"${APP}/nic_${ASSY}.nffw\" \"${FW}\"\n\n# insert distro-specific initramfs section here...\n</code></pre> <p>\u5207\u6362\u56fa\u4ef6\u9009\u9879\u5e76\u91cd\u8f7d\u9a71\u52a8\uff1a</p> <pre><code>./agilio-tc-fw-select.sh ens47np0 scan\nrmmod nfp\nmodprobe nfp\n</code></pre> <p>\u68c0\u67e5\u53ef\u7528 VF \u6570\u91cf\uff0c\u5e76\u521b\u5efa VF\uff1a</p> <pre><code># cat /sys/class/net/ens3/device/sriov_totalvfs\n65\n\n# echo 4 &gt; /sys/class/net/ens47/device/sriov_numvfs\n</code></pre>"},{"location":"advance/offload-corigine/#sr-iov-device-plugin","title":"\u5b89\u88c5 SR-IOV Device Plugin","text":"<p>\u7531\u4e8e\u6bcf\u4e2a\u673a\u5668\u7684 VF \u6570\u91cf\u6709\u9650\uff0c\u6bcf\u4e2a\u4f7f\u7528\u52a0\u901f\u7684 Pod \u4f1a\u5360\u7528 VF \u8d44\u6e90\uff0c\u6211\u4eec\u9700\u8981\u4f7f\u7528 SR-IOV Device Plugin \u7ba1\u7406\u76f8\u5e94\u8d44\u6e90\uff0c\u4f7f\u5f97\u8c03\u5ea6\u5668\u77e5\u9053\u5982\u4f55\u6839\u636e \u8d44\u6e90\u8fdb\u884c\u8c03\u5ea6\u3002</p> <p>\u521b\u5efa SR-IOV \u76f8\u5173 Configmap\uff1a</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: sriovdp-config\nnamespace: kube-system\ndata:\nconfig.json: |\n{\n\"resourceList\": [{\n\"resourcePrefix\": \"corigine.com\",\n\"resourceName\": \"agilio_sriov\",\n\"selectors\": {\n\"vendors\": [\"19ee\"],\n\"devices\": [\"6003\"],\n\"drivers\": [\"nfp_netvf\"]\n}\n}\n]\n}\n</code></pre> <p>\u53c2\u8003 SR-IOV \u6587\u6863\u8fdb\u884c\u90e8\u7f72:</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/intel/sriov-network-device-plugin/master/deployments/k8s-v1.16/sriovdp-daemonset.yaml\n</code></pre> <p>\u68c0\u67e5 SR-IOV \u8d44\u6e90\u662f\u5426\u5df2\u7ecf\u6ce8\u518c\u5230 Kubernetes Node \u4e2d\uff1a</p> <pre><code>kubectl describe no containerserver  | grep corigine\n\ncorigine.com/agilio_sriov:  4\ncorigine.com/agilio_sriov:  4\ncorigine.com/agilio_sriov  0           0\n</code></pre>"},{"location":"advance/offload-corigine/#multus-cni","title":"\u5b89\u88c5 Multus-CNI","text":"<p>SR-IOV Device Plugin \u8c03\u5ea6\u65f6\u83b7\u5f97\u7684\u8bbe\u5907 ID \u9700\u8981\u901a\u8fc7 Multus-CNI \u4f20\u9012\u7ed9 Kube-OVN\uff0c\u56e0\u6b64\u9700\u8981\u914d\u7f6e Multus-CNI \u914d\u5408\u5b8c\u6210\u76f8\u5173\u4efb\u52a1\u3002</p> <p>\u53c2\u8003 Multus-CNI \u6587\u6863\u8fdb\u884c\u90e8\u7f72\uff1a</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml\n</code></pre> <p>\u521b\u5efa <code>NetworkAttachmentDefinition</code>\uff1a</p> <pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: default\nnamespace: default\nannotations:\nk8s.v1.cni.cncf.io/resourceName: corigine.com/agilio_sriov\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.1\",\n\"name\": \"kube-ovn\",\n\"plugins\":[\n{\n\"type\":\"kube-ovn\",\n\"server_socket\":\"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"default.default.ovn\"\n},\n{\n\"type\":\"portmap\",\n\"capabilities\":{\n\"portMappings\":true\n}\n}\n]\n}'\n</code></pre> <ul> <li><code>provider</code>: \u683c\u5f0f\u4e3a\u5f53\u524d <code>NetworkAttachmentDefinition</code> \u7684 {name}.{namespace}.ovn\u3002</li> </ul>"},{"location":"advance/offload-corigine/#kube-ovn","title":"Kube-OVN \u4e2d\u5f00\u542f\u5378\u8f7d\u6a21\u5f0f","text":"<p>\u4e0b\u8f7d\u5b89\u88c5\u811a\u672c\uff1a</p> <pre><code>wget https://raw.githubusercontent.com/alauda/kube-ovn/master/dist/images/install.sh\n</code></pre> <p>\u4fee\u6539\u76f8\u5173\u53c2\u6570\uff0c<code>IFACE</code> \u9700\u8981\u4e3a\u7269\u7406\u7f51\u5361\u540d\uff0c\u8be5\u7f51\u5361\u9700\u8981\u6709\u53ef\u8def\u7531 IP\uff1a</p> <pre><code>ENABLE_MIRROR=${ENABLE_MIRROR:-false}\nHW_OFFLOAD=${HW_OFFLOAD:-true}\nENABLE_LB=${ENABLE_LB:-false}\nIFACE=\"ensp01\"\n</code></pre> <p>\u5b89\u88c5 Kube-OVN\uff1a</p> <pre><code>bash install.sh\n</code></pre>"},{"location":"advance/offload-corigine/#vf-pod","title":"\u521b\u5efa\u4f7f\u7528 VF \u7f51\u5361\u7684 Pod","text":"<p>\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b yaml \u683c\u5f0f\u521b\u5efa\u4f7f\u7528 VF \u8fdb\u884c\u7f51\u7edc\u5378\u8f7d\u52a0\u901f\u7684 Pod:</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: nginx\nnamespace: default\nannotations:\nv1.multus-cni.io/default-network: default/default\nspec:\ncontainers:\n- name: nginx\nimage: docker.io/library/nginx:alpine\nresources:\nrequests:\ncorigine.com/agilio_sriov: '1'\nlimits:\ncorigine.com/agilio_sriov: '1'\n</code></pre> <ul> <li><code>v1.multus-cni.io/default-network</code>: \u4e3a\u4e0a\u4e00\u6b65\u9aa4\u4e2d <code>NetworkAttachmentDefinition</code> \u7684 {namespace}/{name}\u3002</li> </ul> <p>\u53ef\u901a\u8fc7\u5728 Pod \u8fd0\u884c\u8282\u70b9\u7684 <code>ovs-ovn</code> \u5bb9\u5668\u4e2d\u8fd0\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u89c2\u5bdf\u5378\u8f7d\u662f\u5426\u6210\u529f\uff1a</p> <pre><code># ovs-appctl dpctl/dump-flows -m type=offloaded\nufid:91cc45de-e7e9-4935-8f82-1890430b0f66, skb_priority(0/0),skb_mark(0/0),ct_state(0/0x23),ct_zone(0/0),ct_mark(0/0),ct_label(0/0x1),recirc_id(0),dp_hash(0/0),in_port(5b45c61b307e_h),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:c5:6d:4e,dst=00:00:00:e7:16:ce),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:941539, bytes:62142230, used:0.260s, offloaded:yes, dp:tc, actions:54235e5753b8_h\nufid:e00768d7-e652-4d79-8182-3291d852b791, skb_priority(0/0),skb_mark(0/0),ct_state(0/0x23),ct_zone(0/0),ct_mark(0/0),ct_label(0/0x1),recirc_id(0),dp_hash(0/0),in_port(54235e5753b8_h),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:e7:16:ce,dst=00:00:00:c5:6d:4e),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:82386659, bytes:115944854173, used:0.260s, offloaded:yes, dp:tc, actions:5b45c61b307e_h\n</code></pre> <p>\u5982\u679c\u6709 <code>offloaded:yes, dp:tc</code> \u5185\u5bb9\u8bc1\u660e\u5378\u8f7d\u6210\u529f\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/offload-mellanox/","title":"Mellanox \u7f51\u5361 Offload \u652f\u6301","text":"<p>Kube-OVN \u5728\u6700\u7ec8\u7684\u6570\u636e\u5e73\u9762\u4f7f\u7528 OVS \u6765\u5b8c\u6210\u6d41\u91cf\u8f6c\u53d1\uff0c\u76f8\u5173\u7684\u6d41\u8868\u5339\u914d\uff0c\u96a7\u9053\u5c01\u88c5\u7b49\u529f\u80fd\u4e3a CPU \u5bc6\u96c6\u578b\uff0c\u5728\u5927\u6d41\u91cf\u4e0b\u4f1a\u6d88\u8017\u5927\u91cf CPU \u8d44\u6e90\u5e76\u5bfc\u81f4 \u5ef6\u8fdf\u4e0a\u5347\u548c\u541e\u5410\u91cf\u4e0b\u964d\u3002Mellanox \u7684 Accelerated Switching And Packet Processing (ASAP\u00b2) \u6280\u672f\u53ef\u4ee5\u5c06 OVS \u76f8\u5173\u7684\u64cd\u4f5c\u5378\u8f7d\u5230\u786c\u4ef6\u7f51\u5361\u5185\u7684 eSwitch \u4e0a\u6267\u884c\u3002\u8be5\u6280\u672f\u53ef\u4ee5\u5728\u65e0\u9700\u5bf9 OVS \u63a7\u5236\u5e73\u9762\u8fdb\u884c\u4fee\u6539\u7684\u60c5\u51b5\u4e0b\uff0c\u7f29\u77ed\u6570\u636e\u8def\u5f84\uff0c\u907f\u514d\u5bf9\u4e3b\u673a CPU \u8d44\u6e90\u7684\u4f7f\u7528\uff0c\u5927\u5e45\u964d\u4f4e\u5ef6\u8fdf\u5e76\u663e\u8457\u63d0\u5347\u541e\u5410\u91cf\u3002</p> <p></p>"},{"location":"advance/offload-mellanox/#_1","title":"\u524d\u7f6e\u6761\u4ef6","text":"<ul> <li>Mellanox CX5/CX6/CX7/BlueField \u7b49\u652f\u6301 ASAP\u00b2 \u7684\u786c\u4ef6\u7f51\u5361\u3002</li> <li>CentOS 8 Stream \u6216\u4e0a\u6e38 Linux 5.7 \u4ee5\u4e0a\u5185\u6838\u652f\u6301\u3002</li> <li>\u7531\u4e8e\u5f53\u524d\u7f51\u5361\u4e0d\u652f\u6301 <code>dp_hash</code> \u548c <code>hash</code> \u64cd\u4f5c\u5378\u8f7d\uff0c\u9700\u5173\u95ed OVN LB \u529f\u80fd\u3002</li> <li>\u4e3a\u4e86\u652f\u6301\u5378\u8f7d\u6a21\u5f0f\uff0c\u7f51\u5361\u4e0d\u80fd\u505a bond\u3002</li> </ul>"},{"location":"advance/offload-mellanox/#sr-iov-device-plugin","title":"\u914d\u7f6e SR-IOV \u548c Device Plugin","text":"<p>Mellanox \u7f51\u5361\u652f\u6301\u4e24\u79cd\u914d\u7f6e offload \u7684\u65b9\u5f0f\uff0c\u4e00\u79cd\u624b\u52a8\u914d\u7f6e\u7f51\u5361 SR-IOV \u548c Device Plugin\uff0c\u53e6\u4e00\u79cd\u901a\u8fc7 sriov-network-operator \u8fdb\u884c\u81ea\u52a8\u914d\u7f6e\u3002</p>"},{"location":"advance/offload-mellanox/#sr-iov-device-plugin_1","title":"\u624b\u52a8\u914d\u7f6e SR-IOV \u548c Device Plugin","text":"<p>\u67e5\u8be2\u7f51\u5361\u7684\u8bbe\u5907 ID\uff0c\u4e0b\u9762\u7684\u4f8b\u5b50\u4e2d\u4e3a <code>42:00.0</code>\uff1a</p> <pre><code># lspci -nn | grep ConnectX-5\n42:00.0 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5] [15b3:1017]\n</code></pre> <p>\u6839\u636e\u8bbe\u5907 ID \u627e\u5230\u5bf9\u5e94\u7f51\u5361\uff1a</p> <pre><code># ls -l /sys/class/net/ | grep 42:00.0\nlrwxrwxrwx. 1 root root 0 Jul 22 23:16 p4p1 -&gt; ../../devices/pci0000:40/0000:40:02.0/0000:42:00.0/net/p4p1\n</code></pre> <p>\u68c0\u67e5\u53ef\u7528 VF \u6570\u91cf\uff1a</p> <pre><code># cat /sys/class/net/p4p1/device/sriov_totalvfs\n8\n</code></pre> <p>\u521b\u5efa VF\uff0c\u603b\u6570\u4e0d\u8981\u8d85\u8fc7\u4e0a\u9762\u67e5\u8be2\u51fa\u7684\u6570\u91cf\uff1a</p> <pre><code># echo '4' &gt; /sys/class/net/p4p1/device/sriov_numvfs\n# ip link show p4p1\n10: p4p1: &lt;NO-CARRIER,BROADCAST,MULTICAST,UP&gt; mtu 1500 qdisc mq state DOWN mode DEFAULT group default qlen 1000\nlink/ether b8:59:9f:c1:ec:12 brd ff:ff:ff:ff:ff:ff\n    vf 0 MAC 00:00:00:00:00:00, spoof checking off, link-state auto, trust off, query_rss off\n    vf 1 MAC 00:00:00:00:00:00, spoof checking off, link-state auto, trust off, query_rss off\n    vf 2 MAC 00:00:00:00:00:00, spoof checking off, link-state auto, trust off, query_rss off\n    vf 3 MAC 00:00:00:00:00:00, spoof checking off, link-state auto, trust off, query_rss off\n# ip link set p4p1 up\n</code></pre> <p>\u627e\u5230\u4e0a\u8ff0 VF \u5bf9\u5e94\u7684\u8bbe\u5907 ID\uff1a</p> <pre><code># lspci -nn | grep ConnectX-5\n42:00.0 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5] [15b3:1017]\n42:00.1 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5] [15b3:1017]\n42:00.2 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n42:00.3 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n42:00.4 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n42:00.5 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n</code></pre> <p>\u5c06 VF \u4ece\u9a71\u52a8\u4e2d\u89e3\u7ed1\uff1a</p> <pre><code>echo 0000:42:00.2 &gt; /sys/bus/pci/drivers/mlx5_core/unbind\necho 0000:42:00.3 &gt; /sys/bus/pci/drivers/mlx5_core/unbind\necho 0000:42:00.4 &gt; /sys/bus/pci/drivers/mlx5_core/unbind\necho 0000:42:00.5 &gt; /sys/bus/pci/drivers/mlx5_core/unbind\n</code></pre> <p>\u5f00\u542f eSwitch \u6a21\u5f0f\uff0c\u5e76\u8bbe\u7f6e\u786c\u4ef6\u5378\u8f7d\uff1a</p> <pre><code>devlink dev eswitch set pci/0000:42:00.0 mode switchdev\nethtool -K enp66s0f0 hw-tc-offload on\n</code></pre> <p>\u91cd\u65b0\u7ed1\u5b9a\u9a71\u52a8\uff0c\u5b8c\u6210 VF \u8bbe\u7f6e\uff1a</p> <pre><code>echo 0000:42:00.2 &gt; /sys/bus/pci/drivers/mlx5_core/bind\necho 0000:42:00.3 &gt; /sys/bus/pci/drivers/mlx5_core/bind\necho 0000:42:00.4 &gt; /sys/bus/pci/drivers/mlx5_core/bind\necho 0000:42:00.5 &gt; /sys/bus/pci/drivers/mlx5_core/bind\n</code></pre> <p><code>NetworkManager</code> \u7684\u4e00\u4e9b\u884c\u4e3a\u53ef\u80fd\u4f1a\u5bfc\u81f4\u9a71\u52a8\u5f02\u5e38\uff0c\u5982\u679c\u5378\u8f7d\u51fa\u73b0\u95ee\u9898\u5efa\u8bae\u5173\u95ed <code>NetworkManager</code> \u518d\u8fdb\u884c\u5c1d\u8bd5\uff1a</p> <pre><code>systemctl stop NetworkManager\nsystemctl disable NetworkManager\n</code></pre> <p>\u7531\u4e8e\u6bcf\u4e2a\u673a\u5668\u7684 VF \u6570\u91cf\u4f18\u5148\uff0c\u6bcf\u4e2a\u4f7f\u7528\u52a0\u901f\u7684 Pod \u4f1a\u5360\u7528 VF \u8d44\u6e90\uff0c\u6211\u4eec\u9700\u8981\u4f7f\u7528 SR-IOV Device Plugin \u7ba1\u7406\u76f8\u5e94\u8d44\u6e90\uff0c\u4f7f\u5f97\u8c03\u5ea6\u5668\u77e5\u9053\u5982\u4f55\u6839\u636e \u8d44\u6e90\u8fdb\u884c\u8c03\u5ea6\u3002</p> <p>\u521b\u5efa SR-IOV \u76f8\u5173 Configmap\uff1a</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: sriovdp-config\nnamespace: kube-system\ndata:\nconfig.json: |\n{\n\"resourceList\": [{\n\"resourcePrefix\": \"mellanox.com\",\n\"resourceName\": \"cx5_sriov_switchdev\",\n\"selectors\": {\n\"vendors\": [\"15b3\"],\n\"devices\": [\"1018\"],\n\"drivers\": [\"mlx5_core\"]\n}\n}\n]\n}\n</code></pre> <p>\u53c2\u8003 SR-IOV \u6587\u6863\u8fdb\u884c\u90e8\u7f72:</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/intel/sriov-network-device-plugin/master/deployments/k8s-v1.16/sriovdp-daemonset.yaml\n</code></pre> <p>\u68c0\u67e5 SR-IOV \u8d44\u6e90\u662f\u5426\u5df2\u7ecf\u6ce8\u518c\u5230 Kubernetes Node \u4e2d\uff1a</p> <pre><code>kubectl describe node kube-ovn-01  | grep mellanox\n\nmellanox.com/cx5_sriov_switchdev:  4\nmellanox.com/cx5_sriov_switchdev:  4\nmellanox.com/cx5_sriov_switchdev  0           0\n</code></pre>"},{"location":"advance/offload-mellanox/#sriov-network-operator-sr-iov-device-plugin","title":"\u4f7f\u7528 sriov-network-operator \u914d\u7f6e SR-IOV \u548c Device Plugin","text":"<p>\u5b89\u88c5 node-feature-discovery \u81ea\u52a8\u68c0\u6d4b\u786c\u4ef6\u7684\u529f\u80fd\u548c\u7cfb\u7edf\u914d\u7f6e:</p> <pre><code>kubectl apply -k https://github.com/kubernetes-sigs/node-feature-discovery/deployment/overlays/default?ref=v0.11.3\n</code></pre> <p>\u6216\u8005\u901a\u8fc7\u4e0b\u9762\u7684\u547d\u4ee4\uff0c\u624b\u52a8\u7ed9\u6709 offload \u80fd\u529b\u7684\u7f51\u5361\u589e\u52a0 annotation:</p> <pre><code>kubectl label nodes [offloadNicNode] feature.node.kubernetes.io/network-sriov.capable=true\n</code></pre> <p>\u514b\u9686\u4ee3\u7801\u4ed3\u5e93\u5e76\u5b89\u88c5 Operator\uff1a</p> <pre><code>git clone --depth=1 https://github.com/kubeovn/sriov-network-operator.git\nkubectl apply -k sriov-network-operator/deploy\n</code></pre> <p>\u68c0\u67e5 Operator \u7ec4\u4ef6\u662f\u5426\u5de5\u4f5c\u6b63\u5e38\uff1a</p> <pre><code># kubectl get -n kube-system all | grep sriov\nNAME                                          READY   STATUS    RESTARTS   AGE\npod/sriov-network-config-daemon-bf9nt         1/1     Running   0          8s\npod/sriov-network-operator-54d7545f65-296gb   1/1     Running   0          10s\n\nNAME                                         DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR                                                 AGE\ndaemonset.apps/sriov-network-config-daemon   1         1         1       1            1           beta.kubernetes.io/os=linux,feature.node.kubernetes.io/network-sriov.capable=true   8s\n\nNAME                                     READY   UP-TO-DATE   AVAILABLE   AGE\ndeployment.apps/sriov-network-operator   1/1     1            1           10s\n\nNAME                                                DESIRED   CURRENT   READY   AGE\nreplicaset.apps/sriov-network-operator-54d7545f65   1         1         1       10s\n</code></pre> <p>\u68c0\u67e5 <code>SriovNetworkNodeState</code>\uff0c\u4e0b\u9762\u4ee5 <code>node1</code> \u8282\u70b9\u4e3a\u4f8b\uff0c\u8be5\u8282\u70b9\u4e0a\u6709\u4e24\u4e2a Mellanox \u7f51\u5361\uff1a</p> <pre><code># kubectl get sriovnetworknodestates.sriovnetwork.openshift.io -n kube-system node1 -o yaml\napiVersion: sriovnetwork.openshift.io/v1\nkind: SriovNetworkNodeState\nspec: ...\nstatus:\n  interfaces:\n  - deviceID: \"1017\"\ndriver: mlx5_core\n    mtu: 1500\npciAddress: \"0000:5f:00.0\"\ntotalvfs: 8\nvendor: \"15b3\"\nlinkSeed: 25000Mb/s\n    linkType: ETH\n    mac: 08:c0:eb:f4:85:bb\n    name: ens41f0np0\n  - deviceID: \"1017\"\ndriver: mlx5_core\n    mtu: 1500\npciAddress: \"0000:5f:00.1\"\ntotalvfs: 8\nvendor: \"15b3\"\nlinkSeed: 25000Mb/s\n    linkType: ETH\n    mac: 08:c0:eb:f4:85:bb\n    name: ens41f1np1\n</code></pre> <p>\u521b\u5efa <code>SriovNetworkNodePolicy</code> \u8d44\u6e90\uff0c\u5e76\u901a\u8fc7 <code>nicSelector</code> \u9009\u62e9\u8981\u7ba1\u7406\u7684\u7f51\u5361\uff1a</p> <pre><code>apiVersion: sriovnetwork.openshift.io/v1\nkind: SriovNetworkNodePolicy\nmetadata:\nname: policy\nnamespace: kube-system\nspec:\nnodeSelector:\nfeature.node.kubernetes.io/network-sriov.capable: \"true\"\neSwitchMode: switchdev\nnumVfs: 3\nnicSelector:\npfNames:\n- ens41f0np0\n- ens41f1np1\nresourceName: cx_sriov_switchdev\n</code></pre> <p>\u518d\u6b21\u68c0\u67e5 <code>SriovNetworkNodeState</code> \u7684 <code>status</code> \u5b57\u6bb5\uff1a</p> <pre><code># kubectl get sriovnetworknodestates.sriovnetwork.openshift.io -n kube-system node1 -o yaml\n\n...\nspec:\n  interfaces:\n  - eSwitchMode: switchdev\n    name: ens41f0np0\n    numVfs: 3\npciAddress: 0000:5f:00.0\n    vfGroups:\n    - policyName: policy\n      vfRange: 0-2\n      resourceName: cx_sriov_switchdev\n  - eSwitchMode: switchdev\n    name: ens41f1np1\n    numVfs: 3\npciAddress: 0000:5f:00.1\n    vfGroups:\n    - policyName: policy\n      vfRange: 0-2\n      resourceName: cx_sriov_switchdev\nstatus:\n  interfaces\n  - Vfs:\n    - deviceID: 1018\ndriver: mlx5_core\n      pciAddress: 0000:5f:00.2\n      vendor: \"15b3\"\n- deviceID: 1018\ndriver: mlx5_core\n      pciAddress: 0000:5f:00.3\n      vendor: \"15b3\"\n- deviceID: 1018\ndriver: mlx5_core\n      pciAddress: 0000:5f:00.4\n      vendor: \"15b3\"\ndeviceID: \"1017\"\ndriver: mlx5_core\n    linkSeed: 25000Mb/s\n    linkType: ETH\n    mac: 08:c0:eb:f4:85:ab\n    mtu: 1500\nname: ens41f0np0\n    numVfs: 3\npciAddress: 0000:5f:00.0\n    totalvfs: 3\nvendor: \"15b3\"\n- Vfs:\n    - deviceID: 1018\ndriver: mlx5_core\n      pciAddress: 0000:5f:00.5\n      vendor: \"15b3\"\n- deviceID: 1018\ndriver: mlx5_core\n      pciAddress: 0000:5f:00.6\n      vendor: \"15b3\"\n- deviceID: 1018\ndriver: mlx5_core\n      pciAddress: 0000:5f:00.7\n      vendor: \"15b3\"\ndeviceID: \"1017\"\ndriver: mlx5_core\n    linkSeed: 25000Mb/s\n    linkType: ETH\n    mac: 08:c0:eb:f4:85:bb\n    mtu: 1500\nname: ens41f1np1\n    numVfs: 3\npciAddress: 0000:5f:00.1\n    totalvfs: 3\nvendor: \"15b3\"\n</code></pre> <p>\u68c0\u67e5 VF \u7684\u72b6\u6001\uff1a</p> <pre><code># lspci -nn | grep ConnectX\n5f:00.0 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5] [15b3:1017]\n5f:00.1 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5] [15b3:1017]\n5f:00.2 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n5f:00.3 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n5f:00.4 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n5f:00.5 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n5f:00.6 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n5f:00.7 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n</code></pre> <p>\u68c0\u67e5 PF \u5de5\u4f5c\u6a21\u5f0f\uff1a</p> <pre><code># cat /sys/class/net/ens41f0np0/compat/devlink/mode\nswitchdev\n</code></pre>"},{"location":"advance/offload-mellanox/#multus-cni","title":"\u5b89\u88c5 Multus-CNI","text":"<p>SR-IOV Device Plugin \u8c03\u5ea6\u65f6\u83b7\u5f97\u7684\u8bbe\u5907 ID \u9700\u8981\u901a\u8fc7 Multus-CNI \u4f20\u9012\u7ed9 Kube-OVN\uff0c\u56e0\u6b64\u9700\u8981\u914d\u7f6e Multus-CNI \u914d\u5408\u5b8c\u6210\u76f8\u5173\u4efb\u52a1\u3002</p> <p>\u53c2\u8003 Multus-CNI \u6587\u6863\u8fdb\u884c\u90e8\u7f72\uff1a</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml\n</code></pre> <p>\u521b\u5efa <code>NetworkAttachmentDefinition</code>\uff1a</p> <pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: default\nnamespace: default\nannotations:\nk8s.v1.cni.cncf.io/resourceName: mellanox.com/cx5_sriov_switchdev\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.1\",\n\"name\": \"kube-ovn\",\n\"plugins\":[\n{\n\"type\":\"kube-ovn\",\n\"server_socket\":\"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"default.default.ovn\"\n},\n{\n\"type\":\"portmap\",\n\"capabilities\":{\n\"portMappings\":true\n}\n}\n]\n}'\n</code></pre> <ul> <li><code>provider</code>: \u683c\u5f0f\u4e3a\u5f53\u524d <code>NetworkAttachmentDefinition</code> \u7684 {name}.{namespace}.ovn\u3002</li> </ul>"},{"location":"advance/offload-mellanox/#kube-ovn","title":"Kube-OVN \u4e2d\u5f00\u542f\u5378\u8f7d\u6a21\u5f0f","text":"<p>\u4e0b\u8f7d\u5b89\u88c5\u811a\u672c\uff1a</p> <pre><code>wget https://raw.githubusercontent.com/alauda/kube-ovn/master/dist/images/install.sh\n</code></pre> <p>\u4fee\u6539\u76f8\u5173\u53c2\u6570\uff0c<code>IFACE</code> \u9700\u8981\u4e3a\u7269\u7406\u7f51\u5361\u540d\uff0c\u8be5\u7f51\u5361\u9700\u8981\u6709\u53ef\u8def\u7531 IP\uff1a</p> <pre><code>ENABLE_MIRROR=${ENABLE_MIRROR:-false}\nHW_OFFLOAD=${HW_OFFLOAD:-true}\nENABLE_LB=${ENABLE_LB:-false}\nIFACE=\"ensp01\"\n</code></pre> <p>\u5b89\u88c5 Kube-OVN\uff1a</p> <pre><code>bash install.sh\n</code></pre>"},{"location":"advance/offload-mellanox/#vf-pod","title":"\u521b\u5efa\u4f7f\u7528 VF \u7f51\u5361\u7684 Pod","text":"<p>\u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b yaml \u683c\u5f0f\u521b\u5efa\u4f7f\u7528 VF \u8fdb\u884c\u7f51\u7edc\u5378\u8f7d\u52a0\u901f\u7684 Pod:</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: nginx\nannotations:\nv1.multus-cni.io/default-network: default/default\nspec:\ncontainers:\n- name: nginx\nimage: docker.io/library/nginx:alpine\nresources:\nrequests:\nmellanox.com/cx5_sriov_switchdev: '1'\nlimits:\nmellanox.com/cx5_sriov_switchdev: '1'\n</code></pre> <ul> <li><code>v1.multus-cni.io/default-network</code>: \u4e3a\u4e0a\u4e00\u6b65\u9aa4\u4e2d <code>NetworkAttachmentDefinition</code> \u7684 {namespace}/{name}\u3002</li> </ul> <p>\u53ef\u901a\u8fc7\u5728 Pod \u8fd0\u884c\u8282\u70b9\u7684 <code>ovs-ovn</code> \u5bb9\u5668\u4e2d\u8fd0\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u89c2\u5bdf\u5378\u8f7d\u662f\u5426\u6210\u529f\uff1a</p> <pre><code># ovs-appctl dpctl/dump-flows -m type=offloaded\nufid:91cc45de-e7e9-4935-8f82-1890430b0f66, skb_priority(0/0),skb_mark(0/0),ct_state(0/0x23),ct_zone(0/0),ct_mark(0/0),ct_label(0/0x1),recirc_id(0),dp_hash(0/0),in_port(5b45c61b307e_h),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:c5:6d:4e,dst=00:00:00:e7:16:ce),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:941539, bytes:62142230, used:0.260s, offloaded:yes, dp:tc, actions:54235e5753b8_h\nufid:e00768d7-e652-4d79-8182-3291d852b791, skb_priority(0/0),skb_mark(0/0),ct_state(0/0x23),ct_zone(0/0),ct_mark(0/0),ct_label(0/0x1),recirc_id(0),dp_hash(0/0),in_port(54235e5753b8_h),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:e7:16:ce,dst=00:00:00:c5:6d:4e),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:82386659, bytes:115944854173, used:0.260s, offloaded:yes, dp:tc, actions:5b45c61b307e_h\n</code></pre> <p>\u5982\u679c\u6709 <code>offloaded:yes, dp:tc</code> \u5185\u5bb9\u8bc1\u660e\u5378\u8f7d\u6210\u529f\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/overlay-with-route/","title":"Overlay \u4e0b\u8def\u7531\u65b9\u5f0f\u7f51\u7edc\u6253\u901a","text":"<p>\u5728\u4e00\u4e9b\u573a\u666f\u4e0b\uff0c\u7f51\u7edc\u73af\u5883\u4e0d\u652f\u6301 Underlay \u6a21\u5f0f\uff0c\u4f46\u662f\u4f9d\u7136\u9700\u8981 Pod \u80fd\u548c\u5916\u90e8\u8bbe\u65bd\u76f4\u63a5\u901a\u8fc7 IP \u8fdb\u884c\u4e92\u8bbf\uff0c \u8fd9\u65f6\u5019\u53ef\u4ee5\u4f7f\u7528\u8def\u7531\u65b9\u5f0f\u5c06\u5bb9\u5668\u7f51\u7edc\u548c\u5916\u90e8\u8054\u901a\u3002</p> <p>\u8def\u7531\u6a21\u5f0f\u53ea\u652f\u6301\u9ed8\u8ba4 VPC \u4e0b\u7684 Overlay \u7f51\u7edc\u548c\u5916\u90e8\u6253\u901a\uff0c\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\uff0cPod IP \u4f1a\u76f4\u63a5\u8fdb\u5165\u5e95\u5c42\u7f51\u7edc\uff0c\u5e95\u5c42\u7f51\u7edc\u9700\u8981\u653e\u5f00\u5173\u4e8e\u6e90\u5730\u5740\u548c\u76ee\u5730\u5740\u7684 IP \u68c0\u67e5\u3002</p>"},{"location":"advance/overlay-with-route/#_1","title":"\u524d\u63d0\u6761\u4ef6","text":"<ul> <li>\u6b64\u6a21\u5f0f\u4e0b\uff0c\u4e3b\u673a\u9700\u8981\u5f00\u653e <code>ip_forward</code>\u3002</li> <li>\u68c0\u67e5\u4e3b\u673a iptables \u89c4\u5219\u4e2d\u662f\u5426\u5728 forward \u94fe\u4e2d\u662f\u5426\u6709 Drop \u89c4\u5219\uff0c\u9700\u8981\u653e\u884c\u5bb9\u5668\u76f8\u5173\u6d41\u91cf\u3002</li> <li>\u7531\u4e8e\u53ef\u80fd\u5b58\u5728\u975e\u5bf9\u79f0\u8def\u7531\u7684\u60c5\u51b5\uff0c\u4e3b\u673a\u9700\u653e\u884c ct \u72b6\u6001\u4e3a <code>INVALID</code> \u7684\u6570\u636e\u5305\u3002</li> </ul>"},{"location":"advance/overlay-with-route/#_2","title":"\u8bbe\u7f6e\u6b65\u9aa4","text":"<p>\u5bf9\u4e8e\u9700\u8981\u5bf9\u5916\u76f4\u63a5\u8def\u7531\u7684\u5b50\u7f51\uff0c\u9700\u8981\u5c06\u5b50\u7f51\u7684 <code>natOutgoing</code> \u8bbe\u7f6e\u4e3a <code>false</code>\uff0c\u5173\u95ed nat \u6620\u5c04\uff0c\u4f7f\u5f97 Pod IP \u53ef\u4ee5\u76f4\u63a5\u8fdb\u5165\u5916\u90e8\u7f51\u7edc\u3002</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: routed\nspec:\nprotocol: IPv4\ncidrBlock: 10.166.0.0/16\ndefault: false\nexcludeIps:\n- 10.166.0.1\ngateway: 10.166.0.1\ngatewayType: distributed\nnatOutgoing: false\n</code></pre> <p>\u6b64\u65f6\uff0cPod \u7684\u6570\u636e\u5305\u53ef\u4ee5\u901a\u8fc7\u4e3b\u673a\u8def\u7531\u5230\u8fbe\u5bf9\u7aef\u8282\u70b9\uff0c\u4f46\u662f\u5bf9\u7aef\u8282\u70b9\u8fd8\u4e0d\u77e5\u9053\u56de\u7a0b\u6570\u636e\u5305\u5e94\u8be5\u53d1\u9001\u5230\u54ea\u91cc\uff0c\u9700\u8981\u6dfb\u52a0\u56de\u7a0b\u8def\u7531\u3002</p> <p>\u5982\u679c\u5bf9\u7aef\u4e3b\u673a\u548c\u5bb9\u5668\u6240\u5728\u5bbf\u4e3b\u673a\u5728\u540c\u4e00\u4e2a\u4e8c\u5c42\u7f51\u7edc\uff0c\u6211\u4eec\u53ef\u4ee5\u76f4\u63a5\u5728\u5bf9\u7aef\u4e3b\u673a\u6dfb\u52a0\u9759\u6001\u8def\u7531\u5c06\u5bb9\u5668\u7f51\u7edc\u7684\u4e0b\u4e00\u8df3\u6307\u5411 Kubernetes \u96c6\u7fa4\u5185\u7684\u4efb\u610f\u4e00\u53f0\u673a\u5668\u3002</p> <pre><code>ip route add 10.166.0.0/16 via 192.168.2.10 dev eth0\n</code></pre> <p><code>10.166.0.0/16</code> \u4e3a\u5bb9\u5668\u5b50\u7f51\u7f51\u6bb5\uff0c<code>192.168.2.10</code> \u4e3a Kubernetes \u96c6\u7fa4\u5185\u4efb\u610f\u4e00\u4e2a\u8282\u70b9\u3002</p> <p>\u82e5\u5bf9\u7aef\u4e3b\u673a\u548c\u5bb9\u5668\u6240\u5728\u5bbf\u4e3b\u673a\u4e0d\u5728\u540c\u4e00\u4e2a\u4e8c\u5c42\u7f51\u7edc\uff0c\u5219\u9700\u8981\u5728\u8def\u7531\u5668\u4e0a\u914d\u7f6e\u76f8\u5e94\u7684\u89c4\u5219\uff0c\u901a\u8fc7\u8def\u7531\u5668\u8fdb\u884c\u6253\u901a\u3002</p> <p>\u6ce8\u610f\uff1a \u6307\u5b9a\u67d0\u4e2a\u8282\u70b9 IP \u5b58\u5728\u5355\u70b9\u6545\u969c\u7684\u53ef\u80fd\uff0c\u5982\u679c\u5e0c\u671b\u505a\u5230\u5feb\u901f\u7684\u6545\u969c\u5207\u6362\u53ef\u4ee5\u901a\u8fc7 Keepalived \u7ed9\u591a\u4e2a\u8282\u70b9\u8bbe\u7f6e VIP\uff0c\u540c\u65f6\u5c06\u8def\u7531\u7684\u4e0b\u4e00\u8df3\u6307\u5411 VIP\u3002</p> <p>\u5728\u4e00\u4e9b\u865a\u62df\u5316\u73af\u5883\u4e2d\uff0c\u865a\u62df\u7f51\u7edc\u4f1a\u5c06\u975e\u5bf9\u79f0\u6d41\u91cf\u8bc6\u522b\u4e3a\u975e\u6cd5\u6d41\u91cf\u5e76\u4e22\u5f03\u3002 \u6b64\u65f6\u9700\u8981\u5c06 Subnet \u7684 <code>gatewayType</code> \u8c03\u6574\u4e3a <code>centralized</code>\uff0c\u5e76\u5728\u8def\u7531\u8bbe\u7f6e\u65f6\u5c06\u4e0b\u4e00\u8df3\u8bbe\u7f6e\u4e3a <code>gatewayNode</code> \u8282\u70b9\u7684 IP\u3002</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: routed\nspec:\nprotocol: IPv4\ncidrBlock: 10.166.0.0/16\ndefault: false\nexcludeIps:\n- 10.166.0.1\ngateway: 10.166.0.1\ngatewayType: centralized\ngatewayNode: \"node1\"\nnatOutgoing: false\n</code></pre> <p>\u5982\u679c\u5bf9\u4e8e\u90e8\u5206\u6d41\u91cf\uff08\u5982\u8bbf\u95ee\u5916\u7f51\u7684\u6d41\u91cf\uff09\u4ecd\u7136\u5e0c\u671b\u8fdb\u884c nat \u5904\u7406\uff0c\u8bf7\u53c2\u8003\u9ed8\u8ba4 VPC NAT \u7b56\u7565\u89c4\u5219\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/ovn-eip-fip-snat/","title":"OVN EIP FIP SNAT DNAT \u652f\u6301","text":"<pre><code>graph LR\n\n\npod--&gt;vpc1-subnet--&gt;vpc1--&gt;snat--&gt;lrp--&gt;external-subnet--&gt;gw-node-external-nic</code></pre> <p>Pod \u57fa\u4e8e SNAT \u51fa\u516c\u7f51\u7684\u5927\u81f4\u6d41\u7a0b\uff0c\u6700\u540e\u662f\u7ecf\u8fc7\u7f51\u5173\u8282\u70b9\u7684\u516c\u7f51\u7f51\u5361\u3002</p> <pre><code>graph LR\n\n\npod--&gt;vpc1-subnet--&gt;vpc1--&gt;fip--&gt;lrp--&gt;external-subnet--&gt;local-node-external-nic</code></pre> <p>Pod \u57fa\u4e8e FIP \u51fa\u516c\u7f51\u7684\u5927\u81f4\u6d41\u7a0b\uff0c\u6700\u540e\u53ef\u4ee5\u57fa\u4e8e\u672c\u5730\u8282\u70b9\u7684\u516c\u7f51\u7f51\u5361\u51fa\u516c\u7f51\u3002</p> <p>\u8be5\u529f\u80fd\u6240\u652f\u6301\u7684 CRD \u5728\u4f7f\u7528\u4e0a\u5c06\u548c iptable nat gw \u516c\u7f51\u65b9\u6848\u4fdd\u6301\u57fa\u672c\u4e00\u81f4\u3002</p> <ul> <li>ovn eip: \u7528\u4e8e\u516c\u7f51 ip \u5360\u4f4d\uff0c\u4ece underlay provider network vlan subnet \u4e2d\u5206\u914d</li> <li>ovn fip\uff1a \u4e00\u5bf9\u4e00 dnat snat\uff0c\u4e3a vpc \u5185\u7684 ip \u6216\u8005 vip \u63d0\u4f9b\u516c\u7f51\u76f4\u63a5\u8bbf\u95ee\u80fd\u529b</li> <li>ovn snat\uff1a\u6574\u4e2a\u5b50\u7f51\u6216\u8005\u5355\u4e2a vpc \u5185 ip \u53ef\u4ee5\u57fa\u4e8e snat \u8bbf\u95ee\u516c\u7f51</li> <li>ovn dnat\uff1a\u57fa\u4e8e router lb \u5b9e\u73b0, \u57fa\u4e8e\u516c\u7f51 ip + \u7aef\u53e3 \u76f4\u63a5\u8bbf\u95ee vpc \u5185\u7684 \u4e00\u7ec4 endpoints</li> </ul>"},{"location":"advance/ovn-eip-fip-snat/#1","title":"1. \u90e8\u7f72","text":"<p>\u76ee\u524d\u5141\u8bb8\u6240\u6709\uff08\u9ed8\u8ba4\u4ee5\u53ca\u81ea\u5b9a\u4e49\uff09vpc \u4f7f\u7528\u540c\u4e00\u4e2a provider vlan subnet \u8d44\u6e90\uff0c\u540c\u65f6\u517c\u5bb9\u9ed8\u8ba4 VPC EIP/SNAT\u7684\u573a\u666f\u3002</p> <p>\u7c7b\u4f3c neutron ovn\uff0c\u670d\u52a1\u542f\u52a8\u914d\u7f6e\u4e2d\u9700\u8981\u6307\u5b9a provider network \u76f8\u5173\u7684\u914d\u7f6e\uff0c\u4e0b\u8ff0\u7684\u542f\u52a8\u53c2\u6570\u4e5f\u662f\u4e3a\u4e86\u517c\u5bb9 VPC EIP/SNAT \u7684\u5b9e\u73b0\u3002</p> <p>\u90e8\u7f72\u9636\u6bb5\uff0c\u6839\u636e\u5b9e\u9645\u60c5\u51b5\uff0c\u53ef\u80fd\u9700\u8981\u6307\u5b9a\u9ed8\u8ba4\u516c\u7f51\u903b\u8f91\u4ea4\u6362\u673a\u3002 \u5982\u679c\u5b9e\u9645\u4f7f\u7528\u4e2d\u6ca1\u6709 vlan\uff08\u4f7f\u7528 vlan 0\uff09\uff0c\u90a3\u4e48\u4e0b\u8ff0\u542f\u52a8\u53c2\u6570\u65e0\u9700\u914d\u7f6e\u3002</p> <pre><code># \u90e8\u7f72\u7684\u65f6\u5019\u4f60\u9700\u8981\u53c2\u8003\u4ee5\u4e0a\u573a\u666f\uff0c\u6839\u636e\u5b9e\u9645\u60c5\u51b5\uff0c\u6309\u9700\u6307\u5b9a\u5982\u4e0b\u53c2\u6570\n# 1. kube-ovn-controller \u542f\u52a8\u53c2\u6570\u9700\u8981\u914d\u7f6e\uff1a\n- --external-gateway-vlanid=204\n- --external-gateway-switch=external204\n\n# 2. kube-ovn-cni \u542f\u52a8\u53c2\u6570\u9700\u8981\u914d\u7f6e:\n- --external-gateway-switch=external204 ### \u4ee5\u4e0a\u914d\u7f6e\u90fd\u548c\u4e0b\u9762\u7684\u516c\u7f51\u7f51\u7edc\u914d\u7f6e vlan id \u548c\u8d44\u6e90\u540d\u4fdd\u6301\u4e00\u81f4\uff0c\u76ee\u524d\u4ec5\u652f\u6301\u6307\u5b9a\u4e00\u4e2a underlay \u516c\u7f51\u4f5c\u4e3a\u9ed8\u8ba4\u5916\u90e8\u516c\u7f51\u3002\n</code></pre> <p>\u8be5\u914d\u7f6e\u9879\u7684\u8bbe\u8ba1\u548c\u4f7f\u7528\u4e3b\u8981\u8003\u8651\u4e86\u5982\u4e0b\u56e0\u7d20\uff1a</p> <ul> <li>\u57fa\u4e8e\u8be5\u914d\u7f6e\u9879\u53ef\u4ee5\u5bf9\u63a5\u5230 provider network\uff0cvlan\uff0csubnet \u7684\u8d44\u6e90\u3002</li> <li>\u57fa\u4e8e\u8be5\u914d\u7f6e\u9879\u53ef\u4ee5\u5c06\u9ed8\u8ba4 vpc enable_eip_snat \u529f\u80fd\u5bf9\u63a5\u5230\u5df2\u6709\u7684 vlan\uff0csubnet \u8d44\u6e90\uff0c\u540c\u65f6\u652f\u6301\u516c\u7f51 ip \u7684 ipam\u3002</li> <li>\u5982\u679c\u4ec5\u4f7f\u7528\u9ed8\u8ba4 vpc \u7684 enable_eip_snat \u6a21\u5f0f, \u4e14\u4ec5\u4f7f\u7528\u65e7\u7684\u57fa\u4e8e pod annotaion \u7684 fip snat\uff0c\u90a3\u4e48\u8fd9\u4e2a\u914d\u7f6e\u65e0\u9700\u914d\u7f6e\u3002</li> <li>\u57fa\u4e8e\u8be5\u914d\u7f6e\u53ef\u4ee5\u4e0d\u4f7f\u7528\u9ed8\u8ba4 vpc enable_eip_snat \u6d41\u7a0b\uff0c\u4ec5\u901a\u8fc7\u5bf9\u5e94\u5230 vlan\uff0csubnet \u6d41\u7a0b\uff0c\u53ef\u4ee5\u517c\u5bb9\u4ec5\u81ea\u5b9a\u4e49 vpc \u4f7f\u7528 eip snat \u7684\u4f7f\u7528\u573a\u666f\u3002</li> </ul>"},{"location":"advance/ovn-eip-fip-snat/#11-underlay","title":"1.1 \u51c6\u5907 underlay \u516c\u7f51\u7f51\u7edc","text":"<pre><code># \u51c6\u5907 provider-network\uff0c vlan\uff0c subnet\n# cat 01-provider-network.yaml\napiVersion: kubeovn.io/v1\nkind: ProviderNetwork\nmetadata:\n  name: external204\nspec:\n  defaultInterface: vlan\n# cat 02-vlan.yaml\napiVersion: kubeovn.io/v1\nkind: Vlan\nmetadata:\n  name: vlan204\nspec:\n  id: 204\nprovider: external204\n# cat 03-vlan-subnet.yaml\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n  name: external204\nspec:\n  protocol: IPv4\n  cidrBlock: 10.5.204.0/24\n  gateway: 10.5.204.254\n  vlan: vlan204\n  excludeIps:\n  - 10.5.204.1..10.5.204.100\n</code></pre>"},{"location":"advance/ovn-eip-fip-snat/#12-vpc-eip_snat","title":"1.2 \u9ed8\u8ba4 vpc \u542f\u7528 eip_snat","text":"<pre><code># \u542f\u7528\u9ed8\u8ba4 vpc \u548c\u4e0a\u8ff0 underlay \u516c\u7f51 provider subnet \u4e92\u8054\ncat 00-centralized-external-gw-no-ip.yaml\napiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: ovn-external-gw-config\n  namespace: kube-system\ndata:\n  enable-external-gw: \"true\"\nexternal-gw-nodes: \"pc-node-1,pc-node-2,pc-node-3\"\ntype: \"centralized\" external-gw-nic: \"vlan\" # \u7528\u4e8e\u63a5\u5165 ovs \u516c\u7f51\u7f51\u6865\u7684\u7f51\u5361\nexternal-gw-addr: \"10.5.204.254/24\" # underlay \u7269\u7406\u7f51\u5173\u7684 ip\n</code></pre> <p>\u76ee\u524d\u8be5\u529f\u80fd\u5df2\u652f\u6301\u53ef\u4ee5\u4e0d\u6307\u5b9a lrp ip \u548c mac\uff0c\u5df2\u652f\u6301\u81ea\u52a8\u83b7\u53d6\uff0c\u521b\u5efa lrp \u7c7b\u578b\u7684 ovn eip \u8d44\u6e90\u3002</p> <p>\u5982\u679c\u6307\u5b9a\u4e86\uff0c\u5219\u76f8\u5f53\u4e8e\u6307\u5b9a ip \u521b\u5efa lrp \u7c7b\u578b\u7684 ovn-eip\u3002 \u5f53\u7136\u4e5f\u53ef\u4ee5\u63d0\u524d\u624b\u52a8\u521b\u5efa lrp \u7c7b\u578b\u7684 ovn eip\u3002</p>"},{"location":"advance/ovn-eip-fip-snat/#13-vpc-eip-snat-fip","title":"1.3 \u81ea\u5b9a\u4e49 vpc \u542f\u7528 eip snat fip \u529f\u80fd","text":"<pre><code># cat 00-ns.yml\napiVersion: v1\nkind: Namespace\nmetadata:\n  name: vpc1\n\n# cat 01-vpc-ecmp-enable-external-bfd.yml\nkind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\n  name: vpc1\nspec:\n  namespaces:\n  - vpc1\n  enableExternal: true\n# vpc \u542f\u7528 enableExternal \u4f1a\u81ea\u52a8\u521b\u5efa lrp \u5173\u8054\u5230\u4e0a\u8ff0\u6307\u5b9a\u7684\u516c\u7f51\n\n# cat 02-subnet.yml\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n  name: vpc1-subnet1\nspec:\n  cidrBlock: 192.168.0.0/24\n  default: false\ndisableGatewayCheck: false\ndisableInterConnection: true\nenableEcmp: true\ngatewayNode: \"\"\ngatewayType: distributed\n  #gatewayType: centralized\nnatOutgoing: false\nprivate: false\nprotocol: IPv4\n  provider: ovn\n  vpc: vpc1\n  namespaces:\n  - vpc1\n\n# \u8fd9\u91cc\u5b50\u7f51\u548c\u4e4b\u524d\u4f7f\u7528\u5b50\u7f51\u4e00\u6837\uff0c\u8be5\u529f\u80fd\u5728 subnet \u4e0a\u6ca1\u6709\u65b0\u589e\u5c5e\u6027\uff0c\u6ca1\u6709\u4efb\u4f55\u53d8\u66f4\n</code></pre> <p>\u4ee5\u4e0a\u6a21\u677f\u5e94\u7528\u540e\uff0c\u5e94\u8be5\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\u8d44\u6e90\u5b58\u5728</p> <pre><code># k ko nbctl show vpc1\nrouter 87ad06fd-71d5-4ff8-a1f0-54fa3bba1a7f (vpc1)\nport vpc1-vpc1-subnet1\n        mac: \"00:00:00:ED:8E:C7\"\nnetworks: [\"192.168.0.1/24\"]\nport vpc1-external204\n        mac: \"00:00:00:EF:05:C7\"\nnetworks: [\"10.5.204.105/24\"]\ngateway chassis: [7cedd14f-265b-42e5-ac17-e03e7a1f2342 276baccb-fe9c-4476-b41d-05872a94976d fd9f140c-c45d-43db-a6c0-0d4f8ea298dd]\nnat 21d853b0-f7b4-40bd-9a53-31d2e2745739\n        external ip: \"10.5.204.115\"\nlogical ip: \"192.168.0.0/24\"\ntype: \"snat\"\n</code></pre> <pre><code># k ko nbctl lr-route-list vpc1\nIPv4 Routes\nRoute Table &lt;main&gt;:\n                0.0.0.0/0              10.5.204.254 dst-ip\n# \u76ee\u524d\u8be5\u8def\u7531\u5df2\u81ea\u52a8\u7ef4\u62a4\n</code></pre>"},{"location":"advance/ovn-eip-fip-snat/#2-ovn-eip","title":"2. ovn-eip","text":"<p>\u8be5\u529f\u80fd\u548c iptables-eip \u8bbe\u8ba1\u548c\u4f7f\u7528\u65b9\u5f0f\u57fa\u672c\u4e00\u81f4\uff0covn-eip \u76ee\u524d\u6709\u4e09\u79cd type</p> <ul> <li>nat: \u7528\u4e8e ovn dnat\uff0cfip, snat, \u8fd9\u4e9b nat \u7c7b\u578b\u4f1a\u8bb0\u5f55\u5728 status \u4e2d</li> <li>lrp: Resources connected to the public network from a vpc can be used by nat</li> <li>lsp: \u7528\u4e8e ovn \u57fa\u4e8e bfd \u7684 ecmp \u9759\u6001\u8def\u7531\u573a\u666f\uff0c\u5728\u7f51\u5173\u8282\u70b9\u4e0a\u63d0\u4f9b\u4e00\u4e2a ovs internal port \u4f5c\u4e3a ecmp \u8def\u7531\u7684\u4e0b\u4e00\u8df3</li> </ul> <pre><code>---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: eip-static\nspec:\n  externalSubnet: external204\n  type: nat\n\n# \u52a8\u6001\u5206\u914d\u4e00\u4e2a eip \u8d44\u6e90\uff0c\u8be5\u8d44\u6e90\u9884\u7559\u7528\u4e8e fip \u573a\u666f\n</code></pre>"},{"location":"advance/ovn-eip-fip-snat/#21-ovn-fip-pod-fip","title":"2.1 ovn-fip \u4e3a pod \u7ed1\u5b9a\u4e00\u4e2a fip","text":"<pre><code># k get po -o wide -n vpc1 vpc-1-busybox01\nNAME              READY   STATUS    RESTARTS   AGE     IP            NODE\nvpc-1-busybox01   1/1     Running   0          3d15h   192.168.0.2   pc-node-2\n\n# k get ip vpc-1-busybox01.vpc1\nNAME                   V4IP          V6IP   MAC                 NODE        SUBNET\nvpc-1-busybox01.vpc1   192.168.0.2          00:00:00:0A:DD:27   pc-node-2   vpc1-subnet1\n\n---\n\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: eip-static\nspec:\n  externalSubnet: external204\n  type: nat\n\n---\nkind: OvnFip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: eip-static\nspec:\n  ovnEip: eip-static\n  ipName: vpc-1-busybox01.vpc1  # \u6ce8\u610f\u8fd9\u91cc\u662f ip crd \u7684\u540d\u5b57\uff0c\u5177\u6709\u552f\u4e00\u6027\n</code></pre> <pre><code># k get ofip\nNAME          VPC    V4EIP          V4IP          READY   IPTYPE   IPNAME\neip-for-vip   vpc1   10.5.204.106   192.168.0.3   true    vip      test-fip-vip\neip-static    vpc1   10.5.204.101   192.168.0.2   true             vpc-1-busybox01.vpc1\n# k get ofip eip-static\nNAME         VPC    V4EIP          V4IP          READY   IPTYPE   IPNAME\neip-static   vpc1   10.5.204.101   192.168.0.2   true             vpc-1-busybox01.vpc1\n\n[root@pc-node-1 03-cust-vpc]# ping 10.5.204.101\nPING 10.5.204.101 (10.5.204.101) 56(84) bytes of data.\n64 bytes from 10.5.204.101: icmp_seq=2 ttl=62 time=1.21 ms\n64 bytes from 10.5.204.101: icmp_seq=3 ttl=62 time=0.624 ms\n64 bytes from 10.5.204.101: icmp_seq=4 ttl=62 time=0.368 ms\n^C\n--- 10.5.204.101 ping statistics ---\n4 packets transmitted, 3 received, 25% packet loss, time 3049ms\nrtt min/avg/max/mdev = 0.368/0.734/1.210/0.352 ms\n[root@pc-node-1 03-cust-vpc]#\n\n# \u53ef\u4ee5\u770b\u5230\u5728 node ping \u9ed8\u8ba4 vpc \u4e0b\u7684 pod \u7684\u516c\u7f51 ip \u662f\u80fd\u901a\u7684\n</code></pre> <pre><code># \u8be5\u516c\u7f51 ip \u80fd\u901a\u7684\u5173\u952e\u8d44\u6e90\u4e3b\u8981\u5305\u62ec\u4ee5\u4e0b\u90e8\u5206\n# k ko nbctl show vpc1\nrouter 87ad06fd-71d5-4ff8-a1f0-54fa3bba1a7f (vpc1)\nport vpc1-vpc1-subnet1\n        mac: \"00:00:00:ED:8E:C7\"\nnetworks: [\"192.168.0.1/24\"]\nport vpc1-external204\n        mac: \"00:00:00:EF:05:C7\"\nnetworks: [\"10.5.204.105/24\"]\ngateway chassis: [7cedd14f-265b-42e5-ac17-e03e7a1f2342 276baccb-fe9c-4476-b41d-05872a94976d fd9f140c-c45d-43db-a6c0-0d4f8ea298dd]\nnat 813523e7-c68c-408f-bd8c-cba30cb2e4f4\n        external ip: \"10.5.204.101\"\nlogical ip: \"192.168.0.2\"\ntype: \"dnat_and_snat\"\n</code></pre>"},{"location":"advance/ovn-eip-fip-snat/#22-ovn-fip-vip-fip","title":"2.2 ovn-fip \u4e3a vip \u7ed1\u5b9a\u4e00\u4e2a fip","text":"<p>\u4e3a\u4e86\u4fbf\u4e8e\u4e00\u4e9b vip \u573a\u666f\u7684\u4f7f\u7528\uff0c\u6bd4\u5982 kubevirt \u865a\u62df\u673a\u5185\u90e8\u6211\u53ef\u80fd\u4f1a\u4f7f\u7528\u4e00\u4e9b vip \u63d0\u4f9b\u7ed9 keepalived\uff0ckube-vip \u7b49\u573a\u666f\u6765\u4f7f\u7528\uff0c\u540c\u65f6\u652f\u6301\u516c\u7f51\u8bbf\u95ee\u3002</p> <p>\u90a3\u4e48\u53ef\u4ee5\u57fa\u4e8e fip \u7ed1\u5b9a vpc \u5185\u90e8\u7684 vip \u7684\u65b9\u5f0f\u6765\u63d0\u4f9b vip \u7684\u516c\u7f51\u80fd\u529b\u3002</p> <pre><code># \u5148\u521b\u5efa vip\uff0ceip\uff0c\u518d\u5c06 eip \u7ed1\u5b9a\u5230 vip\n# cat vip.yaml\napiVersion: kubeovn.io/v1\nkind: Vip\nmetadata:\n  name: test-fip-vip\nspec:\n  subnet: vpc1-subnet1\n\n# cat 04-fip.yaml\n---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: eip-for-vip\nspec:\n  externalSubnet: external204\n  type: nat\n\n---\nkind: OvnFip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: eip-for-vip\nspec:\n  ovnEip: eip-for-vip\n  ipType: vip         # \u9ed8\u8ba4\u60c5\u51b5\u4e0b fip \u662f\u9762\u5411 pod ip \u7684\uff0c\u8fd9\u91cc\u9700\u8981\u6807\u6ce8\u6307\u5b9a\u5bf9\u63a5\u5230 vip \u8d44\u6e90\nipName: test-fip-vip\n</code></pre> <pre><code># k get ofip\nNAME          VPC    V4EIP          V4IP          READY   IPTYPE   IPNAME\neip-for-vip   vpc1   10.5.204.106   192.168.0.3   true    vip      test-fip-vip\n\n\n[root@pc-node-1 fip-vip]# ping  10.5.204.106\nPING 10.5.204.106 (10.5.204.106) 56(84) bytes of data.\n64 bytes from 10.5.204.106: icmp_seq=1 ttl=62 time=0.694 ms\n64 bytes from 10.5.204.106: icmp_seq=2 ttl=62 time=0.436 ms\n\n# \u5728 node \u4e0a\u662f ping \u5f97\u901a\u7684\n\n\n# pod \u5185\u90e8\u7684 ip \u4f7f\u7528\u65b9\u5f0f\u5927\u81f4\u5c31\u662f\u5982\u4e0b\u8fd9\u79cd\u60c5\u51b5\n\n[root@pc-node-1 fip-vip]# k -n vpc1 exec -it vpc-1-busybox03 -- bash\n[root@vpc-1-busybox03 /]#\n[root@vpc-1-busybox03 /]#\n[root@vpc-1-busybox03 /]# ip a\n1: lo: &lt;LOOPBACK,UP,LOWER_UP&gt; mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000\nlink/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00\n    inet 127.0.0.1/8 scope host lo\n       valid_lft forever preferred_lft forever\n    inet6 ::1/128 scope host\n       valid_lft forever preferred_lft forever\n1568: eth0@if1569: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1500 qdisc noqueue state UP group default\n    link/ether 00:00:00:56:40:e5 brd ff:ff:ff:ff:ff:ff link-netnsid 0\ninet 192.168.0.5/24 brd 192.168.0.255 scope global eth0\n       valid_lft forever preferred_lft forever\n    inet 192.168.0.3/24 scope global secondary eth0  # \u53ef\u4ee5\u770b\u5230 vip \u7684\u914d\u7f6e\nvalid_lft forever preferred_lft forever\n    inet6 fe80::200:ff:fe56:40e5/64 scope link\n       valid_lft forever preferred_lft forever\n\n[root@vpc-1-busybox03 /]# tcpdump -i eth0 host  192.168.0.3 -netvv\ntcpdump: listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes\n00:00:00:ed:8e:c7 &gt; 00:00:00:56:40:e5, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 62, id 44830, offset 0, flags [DF], proto ICMP (1), length 84)\n10.5.32.51 &gt; 192.168.0.3: ICMP echo request, id 177, seq 1, length 64\n00:00:00:56:40:e5 &gt; 00:00:00:ed:8e:c7, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 43962, offset 0, flags [none], proto ICMP (1), length 84)\n192.168.0.3 &gt; 10.5.32.51: ICMP echo reply, id 177, seq 1, length 64\n\n# pod \u5185\u90e8\u53ef\u4ee5\u6293\u5230 fip \u76f8\u5173\u7684 icmp \u5305\n</code></pre>"},{"location":"advance/ovn-eip-fip-snat/#3-ovn-snat","title":"3. ovn-snat","text":""},{"location":"advance/ovn-eip-fip-snat/#31-ovn-snat-subnet-cidr","title":"3.1 ovn-snat \u5bf9\u5e94\u4e00\u4e2a subnet \u7684 cidr","text":"<p>\u8be5\u529f\u80fd\u548c iptables-snat \u8bbe\u8ba1\u548c\u4f7f\u7528\u65b9\u5f0f\u57fa\u672c\u4e00\u81f4</p> <pre><code># cat 03-subnet-snat.yaml\n---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: snat-for-subnet-in-vpc\nspec:\n  externalSubnet: external204\n  type: nat\n\n---\nkind: OvnSnatRule\napiVersion: kubeovn.io/v1\nmetadata:\n  name: snat-for-subnet-in-vpc\nspec:\n  ovnEip: snat-for-subnet-in-vpc\n  vpcSubnet: vpc1-subnet1 # eip \u5bf9\u5e94\u6574\u4e2a\u7f51\u6bb5\n</code></pre>"},{"location":"advance/ovn-eip-fip-snat/#32-ovn-snat-pod-ip","title":"3.2 ovn-snat \u5bf9\u5e94\u5230\u4e00\u4e2a pod ip","text":"<p>\u8be5\u529f\u80fd\u548c iptables-snat \u8bbe\u8ba1\u548c\u4f7f\u7528\u65b9\u5f0f\u57fa\u672c\u4e00\u81f4</p> <pre><code># cat 03-pod-snat.yaml\n---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: snat-for-pod-vpc-ip\nspec:\n  externalSubnet: external204\n  type: nat\n\n---\nkind: OvnSnatRule\napiVersion: kubeovn.io/v1\nmetadata:\n  name: snat01\nspec:\n  ovnEip: snat-for-pod-vpc-ip\n  ipName: vpc-1-busybox02.vpc1 # eip \u5bf9\u5e94\u5355\u4e2a pod ip\n</code></pre> <p>\u4ee5\u4e0a\u8d44\u6e90\u521b\u5efa\u540e\uff0c\u53ef\u4ee5\u770b\u5230 snat \u516c\u7f51\u529f\u80fd\u4f9d\u8d56\u7684\u5982\u4e0b\u8d44\u6e90\u3002</p> <pre><code># kubectl ko nbctl show vpc1\nrouter 87ad06fd-71d5-4ff8-a1f0-54fa3bba1a7f (vpc1)\nport vpc1-vpc1-subnet1\n        mac: \"00:00:00:ED:8E:C7\"\nnetworks: [\"192.168.0.1/24\"]\nport vpc1-external204\n        mac: \"00:00:00:EF:05:C7\"\nnetworks: [\"10.5.204.105/24\"]\ngateway chassis: [7cedd14f-265b-42e5-ac17-e03e7a1f2342 276baccb-fe9c-4476-b41d-05872a94976d fd9f140c-c45d-43db-a6c0-0d4f8ea298dd]\nnat 21d853b0-f7b4-40bd-9a53-31d2e2745739\n        external ip: \"10.5.204.115\"\nlogical ip: \"192.168.0.0/24\"\ntype: \"snat\"\nnat da77a11f-c523-439c-b1d1-72c664196a0f\n        external ip: \"10.5.204.116\"\nlogical ip: \"192.168.0.4\"\ntype: \"snat\"\n</code></pre> <pre><code>[root@pc-node-1 03-cust-vpc]# k get po -A -o wide  | grep busy\nvpc1            vpc-1-busybox01                                 1/1     Running   0                3d15h   192.168.0.2   pc-node-2   &lt;none&gt;           &lt;none&gt;\nvpc1            vpc-1-busybox02                                 1/1     Running   0                17h     192.168.0.4   pc-node-1   &lt;none&gt;           &lt;none&gt;\nvpc1            vpc-1-busybox03                                 1/1     Running   0                17h     192.168.0.5   pc-node-1   &lt;none&gt;           &lt;none&gt;\nvpc1            vpc-1-busybox04                                 1/1     Running   0                17h     192.168.0.6   pc-node-3   &lt;none&gt;           &lt;none&gt;\nvpc1            vpc-1-busybox05                                 1/1     Running   0                17h     192.168.0.7   pc-node-1   &lt;none&gt;           &lt;none&gt;\n\n# k exec -it -n vpc1            vpc-1-busybox04   bash\nkubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.\n[root@vpc-1-busybox04 /]#\n[root@vpc-1-busybox04 /]#\n[root@vpc-1-busybox04 /]# ip a\n1: lo: &lt;LOOPBACK,UP,LOWER_UP&gt; mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000\nlink/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00\n    inet 127.0.0.1/8 scope host lo\n       valid_lft forever preferred_lft forever\n    inet6 ::1/128 scope host\n       valid_lft forever preferred_lft forever\n17095: eth0@if17096: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1500 qdisc noqueue state UP group default\n    link/ether 00:00:00:76:94:55 brd ff:ff:ff:ff:ff:ff link-netnsid 0\ninet 192.168.0.6/24 brd 192.168.0.255 scope global eth0\n       valid_lft forever preferred_lft forever\n    inet6 fe80::200:ff:fe76:9455/64 scope link\n       valid_lft forever preferred_lft forever\n[root@vpc-1-busybox04 /]# ping 223.5.5.5\nPING 223.5.5.5 (223.5.5.5) 56(84) bytes of data.\n64 bytes from 223.5.5.5: icmp_seq=1 ttl=114 time=22.2 ms\n64 bytes from 223.5.5.5: icmp_seq=2 ttl=114 time=21.8 ms\n\n[root@pc-node-1 03-cust-vpc]# k exec -it -n vpc1            vpc-1-busybox02   bash\nkubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.\n[root@vpc-1-busybox02 /]#\n[root@vpc-1-busybox02 /]#\n[root@vpc-1-busybox02 /]#\n[root@vpc-1-busybox02 /]# ip a\n1: lo: &lt;LOOPBACK,UP,LOWER_UP&gt; mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000\nlink/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00\n    inet 127.0.0.1/8 scope host lo\n       valid_lft forever preferred_lft forever\n    inet6 ::1/128 scope host\n       valid_lft forever preferred_lft forever\n1566: eth0@if1567: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1500 qdisc noqueue state UP group default\n    link/ether 00:00:00:0b:e9:d0 brd ff:ff:ff:ff:ff:ff link-netnsid 0\ninet 192.168.0.4/24 brd 192.168.0.255 scope global eth0\n       valid_lft forever preferred_lft forever\n    inet6 fe80::200:ff:fe0b:e9d0/64 scope link\n       valid_lft forever preferred_lft forever\n[root@vpc-1-busybox02 /]# ping 223.5.5.5\nPING 223.5.5.5 (223.5.5.5) 56(84) bytes of data.\n64 bytes from 223.5.5.5: icmp_seq=2 ttl=114 time=22.7 ms\n64 bytes from 223.5.5.5: icmp_seq=3 ttl=114 time=22.6 ms\n64 bytes from 223.5.5.5: icmp_seq=4 ttl=114 time=22.1 ms\n^C\n--- 223.5.5.5 ping statistics ---\n4 packets transmitted, 3 received, 25% packet loss, time 3064ms\nrtt min/avg/max/mdev = 22.126/22.518/22.741/0.278 ms\n\n# \u53ef\u4ee5\u770b\u5230\u4e24\u4e2a pod \u53ef\u4ee5\u5206\u522b\u57fa\u4e8e\u8fd9\u4e24\u79cd snat \u8d44\u6e90\u4e0a\u5916\u7f51\n</code></pre>"},{"location":"advance/ovn-eip-fip-snat/#4-ovn-dnat","title":"4. ovn-dnat","text":""},{"location":"advance/ovn-eip-fip-snat/#41-ovn-dnat-pod-dnat","title":"4.1 ovn-dnat \u4e3a pod \u7ed1\u5b9a\u4e00\u4e2a dnat","text":"<pre><code>kind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-static\nspec:\nexternalSubnet: underlay\n---\nkind: OvnDnatRule\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-dnat\nspec:\novnEip: eip-dnat\nipName: vpc-1-busybox01.vpc1 # \u6ce8\u610f\u8fd9\u91cc\u662f pod ip crd \u7684\u540d\u5b57\uff0c\u5177\u6709\u552f\u4e00\u6027\nprotocol: tcp\ninternalPort: \"22\"\nexternalPort: \"22\"\n</code></pre> <p>OvnDnatRule \u7684\u914d\u7f6e\u4e0e IptablesDnatRule \u7c7b\u4f3c</p> <pre><code># kubectl get oeip eip-dnat\nNAME       V4IP        V6IP   MAC                 TYPE   READY\neip-dnat   10.5.49.4          00:00:00:4D:CE:49   dnat   true\n\n# kubectl get odnat\nNAME                   EIP                    PROTOCOL   V4EIP        V4IP           INTERNALPORT   EXTERNALPORT   IPNAME                                READY\neip-dnat               eip-dnat               tcp        10.5.49.4    192.168.0.3    22             22             vpc-1-busybox01.vpc1                  true\n</code></pre>"},{"location":"advance/ovn-eip-fip-snat/#42-ovn-dnat-vip-dnat","title":"4.2 ovn-dnat \u4e3a vip \u7ed1\u5b9a\u4e00\u4e2a dnat","text":"<pre><code>kind: OvnDnatRule\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-dnat\nspec:\nipType: vip  # \u9ed8\u8ba4\u60c5\u51b5\u4e0b dnat \u662f\u9762\u5411 pod ip \u7684\uff0c\u8fd9\u91cc\u9700\u8981\u6807\u6ce8\u6307\u5b9a\u5bf9\u63a5\u5230 vip \u8d44\u6e90\novnEip: eip-dnat\nipName: test-dnat-vip\nprotocol: tcp\ninternalPort: \"22\"\nexternalPort: \"22\"\n</code></pre> <p>OvnDnatRule \u7684\u914d\u7f6e\u4e0e IptablesDnatRule \u7c7b\u4f3c</p> <pre><code># kubectl get vip test-dnat-vip\nNAME            V4IP          PV4IP   MAC                 PMAC   V6IP   PV6IP   SUBNET         READY\ntest-dnat-vip   192.168.0.4           00:00:00:D0:C0:B5                         vpc1-subnet1   true\n\n# kubectl get oeip eip-dnat\nNAME       V4IP        V6IP   MAC                 TYPE   READY\neip-dnat   10.5.49.4          00:00:00:4D:CE:49   dnat   true\n\n# kubectl get odnat eip-dnat \nNAME       EIP        PROTOCOL   V4EIP       V4IP          INTERNALPORT   EXTERNALPORT   IPNAME          READY\neip-dnat   eip-dnat   tcp        10.5.49.4   192.168.0.4   22             22             test-dnat-vip   true\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/ovn-ipsec/","title":"\u4f7f\u7528 IPsec \u52a0\u5bc6\u8282\u70b9\u95f4\u901a\u4fe1","text":"<p>\u8be5\u529f\u80fd\u4ece v1.10.11 \u548c v1.11.4 \u540e\u5f00\u59cb\u652f\u6301\uff0ckernel \u7248\u672c\u81f3\u5c11\u662f 3.10.0 \u4ee5\u4e0a\u3002</p>"},{"location":"advance/ovn-ipsec/#ipsec_1","title":"\u542f\u52a8 IPsec","text":"<p>\u4ece Kube-OVN \u6e90\u7801\u62f7\u8d1d\u811a\u672c ipsec.sh\uff0c\u6267\u884c\u547d\u4ee4\u5982\u4e0b\uff0c\u8be5\u811a\u672c\u4f1a\u8c03\u7528 ovs-pki \u751f\u6210\u548c\u5206\u914d\u52a0\u5bc6\u9700\u8981\u7684\u8bc1\u4e66\uff1a</p> <pre><code>bash ipsec.sh init\n</code></pre> <p>\u6267\u884c\u5b8c\u6bd5\u540e\uff0c\u8282\u70b9\u4e4b\u95f4\u4f1a\u534f\u5546\u4e00\u6bb5\u65f6\u95f4\u5efa\u7acb IPsec \u96a7\u9053\uff0c\u7ecf\u9a8c\u503c\u662f\u5341\u51e0\u79d2\u5230\u4e00\u5206\u949f\u4e4b\u95f4\uff0c\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0b\u547d\u4ee4\u6765\u67e5\u770b IPsec \u72b6\u6001\uff1a</p> <pre><code># sh ipsec.sh status\nPod {ovs-ovn-d7hdt} ipsec status...\nInterface name: ovn-a4718e-0 v1 (CONFIGURED)\nTunnel Type:    geneve\n  Local IP:       172.18.0.2\n  Remote IP:      172.18.0.4\n  Address Family: IPv4\n  SKB mark:       None\n  Local cert:     /etc/ipsec.d/certs/8aebd9df-46ef-47b9-85e3-73e9a765296d-cert.pem\n  Local name:     8aebd9df-46ef-47b9-85e3-73e9a765296d\n  Local key:      /etc/ipsec.d/private/8aebd9df-46ef-47b9-85e3-73e9a765296d-privkey.pem\n  Remote cert:    None\n  Remote name:    a4718e55-5b85-4f46-90e6-63527d080590\n  CA cert:        /etc/ipsec.d/cacerts/cacert.pem\n  PSK:            None\n  Custom Options: {}\nOfport:         2\nCFM state:      Disabled\nKernel policies installed:\n  src 172.18.0.2/32 dst 172.18.0.4/32 proto udp sport 6081\nsrc 172.18.0.2/32 dst 172.18.0.4/32 proto udp sport 6081\nsrc 172.18.0.2/32 dst 172.18.0.4/32 proto udp dport 6081\nsrc 172.18.0.2/32 dst 172.18.0.4/32 proto udp dport 6081\nKernel security associations installed:\n  sel src 172.18.0.2/32 dst 172.18.0.4/32 proto udp sport 6081\nsel src 172.18.0.4/32 dst 172.18.0.2/32 proto udp dport 6081\nsel src 172.18.0.2/32 dst 172.18.0.4/32 proto udp dport 6081\nsel src 172.18.0.4/32 dst 172.18.0.2/32 proto udp sport 6081\nIPsec connections that are active:\n\n Pod {ovs-ovn-fvbbj} ipsec status...\nInterface name: ovn-8aebd9-0 v1 (CONFIGURED)\nTunnel Type:    geneve\n  Local IP:       172.18.0.4\n  Remote IP:      172.18.0.2\n  Address Family: IPv4\n  SKB mark:       None\n  Local cert:     /etc/ipsec.d/certs/a4718e55-5b85-4f46-90e6-63527d080590-cert.pem\n  Local name:     a4718e55-5b85-4f46-90e6-63527d080590\n  Local key:      /etc/ipsec.d/private/a4718e55-5b85-4f46-90e6-63527d080590-privkey.pem\n  Remote cert:    None\n  Remote name:    8aebd9df-46ef-47b9-85e3-73e9a765296d\n  CA cert:        /etc/ipsec.d/cacerts/cacert.pem\n  PSK:            None\n  Custom Options: {}\nOfport:         1\nCFM state:      Disabled\nKernel policies installed:\n  src 172.18.0.4/32 dst 172.18.0.2/32 proto udp dport 6081\nsrc 172.18.0.4/32 dst 172.18.0.2/32 proto udp dport 6081\nsrc 172.18.0.4/32 dst 172.18.0.2/32 proto udp sport 6081\nsrc 172.18.0.4/32 dst 172.18.0.2/32 proto udp sport 6081\nKernel security associations installed:\n  sel src 172.18.0.4/32 dst 172.18.0.2/32 proto udp dport 6081\nsel src 172.18.0.2/32 dst 172.18.0.4/32 proto udp sport 6081\nsel src 172.18.0.4/32 dst 172.18.0.2/32 proto udp sport 6081\nsel src 172.18.0.2/32 dst 172.18.0.4/32 proto udp dport 6081\nIPsec connections that are active:\n</code></pre> <p>\u5efa\u7acb\u5b8c\u6210\u540e\u53ef\u4ee5\u6293\u5305\u89c2\u5bdf\u62a5\u6587\u5df2\u7ecf\u88ab\u52a0\u5bc6\uff1a</p> <pre><code># tcpdump -i eth0 -nel esp\n10:01:40.349896 IP kube-ovn-worker &gt; kube-ovn-control-plane.kind: ESP(spi=0xcc91322a,seq=0x13d0), length 156\n10:01:40.350015 IP kube-ovn-control-plane.kind &gt; kube-ovn-worker: ESP(spi=0xc8df4221,seq=0x1d37), length 156\n</code></pre> <p>\u5f53\u6267\u884c\u5b8c\u811a\u672c\u540e\uff0c\u53ef\u4ee5\u901a\u8fc7\u6267\u884c\u547d\u4ee4\u5173\u95ed IPsec\uff1a</p> <pre><code># bash ipsec.sh stop\n</code></pre> <p>\u6216\u8005\u6267\u884c\u547d\u4ee4\u518d\u6b21\u6253\u5f00\uff1a</p> <pre><code># bash ipsec.sh start\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/ovn-l3-ha-based-ecmp-with-bfd/","title":"OVN SNAT \u57fa\u4e8e ECMP BFD \u9759\u6001\u8def\u7531\u7684 L3 HA \u652f\u6301","text":"<p>\u81ea\u5b9a\u4e49 vpc \u57fa\u4e8e ovn snat \u540e\u57fa\u4e8e ecmp \u9759\u6001\u8def\u7531\u54c8\u5e0c\u5230\u591a\u4e2a gw node ovnext0 \u7f51\u5361\u51fa\u516c\u7f51</p> <ul> <li>\u652f\u6301\u57fa\u4e8e bfd \u7684\u9ad8\u53ef\u7528</li> <li>\u4ec5\u652f\u6301 hash \u8d1f\u8f7d\u5747\u8861</li> </ul> <pre><code>graph LR\n\npod--&gt;vpc-subnet--&gt;vpc--&gt;snat--&gt;ecmp--&gt;external-subnet--&gt;gw-node1-ovnext0--&gt; node1-external-switch\nexternal-subnet--&gt;gw-node2-ovnext0--&gt; node2-external-switch\nexternal-subnet--&gt;gw-node3-ovnext0--&gt; node3-external-switch</code></pre> <p>\u8be5\u529f\u80fd\u7684\u4f7f\u7528\u65b9\u5f0f\u548covn-eip-fip-snat.md \u57fa\u672c\u4e00\u81f4\uff0c\u4e00\u81f4\u7684\u90e8\u5206\u5305\u62ec install.sh \u7684\u90e8\u7f72\u90e8\u5206\uff0cprovider-network\uff0cvlan\uff0csubnet \u7684\u51c6\u5907\u90e8\u5206\u3002</p> <p>\u81f3\u4e8e\u4e0d\u76f8\u540c\u7684\u90e8\u5206\uff0c\u4f1a\u5728\u4ee5\u4e0b\u90e8\u5206\u5177\u4f53\u9610\u8ff0\uff0c\u4e3b\u8981\u5305\u62ec lsp \u7c7b\u578b\u7684 ovn-eip \u7684\u521b\u5efa\uff0c\u4ee5\u53ca\u57fa\u4e8e vpc enable_bfd \u81ea\u52a8\u7ef4\u62a4 bfd \u4ee5\u53ca ecmp \u9759\u6001\u8def\u7531\u3002</p>"},{"location":"advance/ovn-l3-ha-based-ecmp-with-bfd/#1","title":"1. \u90e8\u7f72","text":""},{"location":"advance/ovn-l3-ha-based-ecmp-with-bfd/#11-underlay","title":"1.1 \u51c6\u5907 underlay \u516c\u7f51\u7f51\u7edc","text":""},{"location":"advance/ovn-l3-ha-based-ecmp-with-bfd/#12-vpc-eip_snat","title":"1.2 \u9ed8\u8ba4 vpc \u542f\u7528 eip_snat","text":""},{"location":"advance/ovn-l3-ha-based-ecmp-with-bfd/#13-vpc-eip-snat-fip","title":"1.3 \u81ea\u5b9a\u4e49 vpc \u542f\u7528 eip snat fip \u529f\u80fd","text":"<p>\u4ee5\u4e0a\u90e8\u5206\u548c ovn-eip-fip-snat.md \u5b8c\u5168\u4e00\u81f4\uff0c\u8fd9\u4e9b\u529f\u80fd\u9a8c\u8bc1\u901a\u8fc7\u540e\uff0c\u53ef\u4ee5\u76f4\u63a5\u57fa\u4e8e\u5982\u4e0b\u65b9\u5f0f\uff0c\u5c06 vpc \u5207\u6362\u5230\u57fa\u4e8e ecmp \u7684 bfd \u9759\u6001\u8def\u7531\uff0c\u5f53\u7136\u4e5f\u53ef\u4ee5\u5207\u56de\u3002</p> <p>\u81ea\u5b9a\u4e49 vpc \u4f7f\u7528\u8be5\u529f\u80fd\u4e4b\u524d\uff0c\u9700\u8981\u5148\u63d0\u4f9b\u597d\u7f51\u5173\u8282\u70b9\uff0c\u81f3\u5c11\u9700\u8981\u63d0\u4f9b 2 \u4e2a\u4ee5\u4e0a\u7f51\u5173\u8282\u70b9\uff0c\u6ce8\u610f\u5f53\u524d\u5b9e\u73b0 ovn-eip \u7684\u540d\u5b57\u5fc5\u987b\u548c\u7f51\u5173\u8282\u70b9\u540d\u4fdd\u6301\u4e00\u81f4\uff0c\u76ee\u524d\u6ca1\u6709\u505a\u8be5\u8d44\u6e90\u7684\u81ea\u52a8\u5316\u7ef4\u62a4\u3002</p> <pre><code># cat gw-node-eip.yaml\n---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\nname: pc-node-1\nspec:\nexternalSubnet: external204\ntype: lsp\n\n---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\nname: pc-node-2\nspec:\nexternalSubnet: external204\ntype: lsp\n\n---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\nname: pc-node-3\nspec:\nexternalSubnet: external204\ntype: lsp\n</code></pre> <p>\u7531\u4e8e\u8fd9\u4e2a\u573a\u666f\u76ee\u524d\u8bbe\u8ba1\u4e0a\u662f\u4f9b vpc ecmp \u51fa\u516c\u7f51\u4f7f\u7528\uff0c\u6240\u4ee5\u4ee5\u4e0a\u5728\u6ca1\u6709 vpc \u542f\u7528 bfd \u7684\u65f6\u5019\uff0c\u5373\u4e0d\u5b58\u5728\u5e26\u6709 enable bfd \u6807\u7b7e\u7684 lrp \u7684 ovn eip \u7684\u65f6\u5019\uff0c\u7f51\u5173\u8282\u70b9\u4e0d\u4f1a\u89e6\u53d1\u521b\u5efa\u7f51\u5173\u7f51\u5361\uff0c\u4e5f\u65e0\u6cd5\u6210\u529f\u542f\u52a8\u5bf9\u7aef bfd \u4f1a\u8bdd\u7684\u76d1\u542c\u3002</p>"},{"location":"advance/ovn-l3-ha-based-ecmp-with-bfd/#2-vpc-ecmp-bfd-l3-ha","title":"2. \u81ea\u5b9a\u4e49 vpc \u542f\u7528 ecmp bfd L3 HA \u516c\u7f51\u529f\u80fd","text":"<pre><code># cat 01-vpc-ecmp-enable-external-bfd.yml\nkind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\n  name: vpc1\nspec:\n  namespaces:\n  - vpc1\n  enableExternal: true\nenableBfd: true # bfd \u5f00\u5173\u53ef\u4ee5\u968f\u610f\u5207\u6362\uff0c\u5f00\u8868\u793a\u542f\u7528 bfd ecmp \u8def\u7531\n#enableBfd: false \n\n\n# cat 02-subnet.yml\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n  name: vpc1-subnet1\nspec:\n  cidrBlock: 192.168.0.0/24\n  default: false\ndisableGatewayCheck: false\ndisableInterConnection: true\nenableEcmp: true  # \u53ea\u9700\u5f00\u542f ecmp\ngatewayNode: \"\"\ngatewayType: distributed\n  #gatewayType: centralized\nnatOutgoing: false\nprivate: false\nprotocol: IPv4\n  provider: ovn\n  vpc: vpc1\n  namespaces:\n  - vpc1\n</code></pre> <p>\u4f7f\u7528\u4e0a\u7684\u6ce8\u610f\u70b9:</p> <ol> <li>\u81ea\u5b9a\u4e49 vpc \u4e0b\u7684 ecmp \u53ea\u7528\u9759\u6001 ecmp bfd \u8def\u7531\uff0cvpc enableBfd \u548c subnet enableEcmp \u540c\u65f6\u5f00\u542f\u7684\u60c5\u51b5\u4e0b\u624d\u4f1a\u751f\u6548\uff0c\u624d\u4f1a\u81ea\u52a8\u7ba1\u7406\u9759\u6001 ecmp bfd \u8def\u7531\u3002</li> <li>\u4e0a\u8ff0\u914d\u7f6e\u5173\u95ed\u7684\u60c5\u51b5\u4e0b\uff0c\u4f1a\u81ea\u52a8\u5207\u56de\u5e38\u89c4\u9ed8\u8ba4\u9759\u6001\u8def\u7531\u3002</li> <li>\u9ed8\u8ba4 vpc \u65e0\u6cd5\u4f7f\u7528\u8be5\u529f\u80fd\uff0c\u4ec5\u652f\u6301\u81ea\u5b9a\u4e49 vpc\uff0c\u9ed8\u8ba4 vpc \u6709\u66f4\u590d\u6742\u7684\u7b56\u7565\u8def\u7531\u4ee5\u53ca snat \u8bbe\u8ba1\u3002</li> <li>\u81ea\u5b9a\u4e49 vpc \u7684 subnet \u7684 enableEcmp \u4ec5\u4f7f\u7528\u9759\u6001\u8def\u7531\uff0c\u7f51\u5173\u7c7b\u578b gatewayType \u6ca1\u6709\u4f5c\u7528\u3002</li> <li>\u5f53\u5173\u95ed EnableExternal \u65f6\uff0cvpc \u5185\u65e0\u6cd5\u901a\u5916\u7f51\u3002</li> <li>\u5f53\u5f00\u542f EnableExternal \u65f6\uff0c\u5173\u95ed EnableBfd \u65f6\uff0c\u4f1a\u57fa\u4e8e\u666e\u901a\u9ed8\u8ba4\u8def\u7531\u4e0a\u5916\u7f51\uff0c\u4e0d\u5177\u5907\u9ad8\u53ef\u7528\u3002</li> </ol> <pre><code># \u4e0a\u8ff0\u6a21\u677f\u5e94\u7528\u540e ovn \u903b\u8f91\u5c42\u5e94\u8be5\u53ef\u4ee5\u770b\u5230\u5982\u4e0b\u8d44\u6e90\n# \u67e5\u770b vpc\n# k get vpc\nNAME          ENABLEEXTERNAL   ENABLEBFD   STANDBY   SUBNETS                                NAMESPACES\novn-cluster   true                         true      [\"external204\",\"join\",\"ovn-default\"]\nvpc1          true             true        true      [\"vpc1-subnet1\"]                       [\"vpc1\"]\n\n# \u9ed8\u8ba4 vpc \u672a\u652f\u6301 ENABLEBFD\n# \u81ea\u5b9a\u4e49 vpc \u5df2\u652f\u6301\u4e14\u5df2\u542f\u7528\n\n\n# 1. \u521b\u5efa\u4e86 bfd \u4f1a\u8bdd\n# k ko nbctl list bfd\n_uuid               : be7df545-2c4c-4751-878f-b3507987f050\ndetect_mult         : 3\ndst_ip              : \"10.5.204.121\"\nexternal_ids        : {}\nlogical_port        : vpc1-external204\nmin_rx              : 100\nmin_tx              : 100\noptions             : {}\nstatus              : up\n\n_uuid               : 684c4489-5b59-4693-8d8c-3beab93f8093\ndetect_mult         : 3\ndst_ip              : \"10.5.204.109\"\nexternal_ids        : {}\nlogical_port        : vpc1-external204\nmin_rx              : 100\nmin_tx              : 100\noptions             : {}\nstatus              : up\n\n_uuid               : f0f62077-2ae9-4e79-b4f8-a446ec6e784c\ndetect_mult         : 3\ndst_ip              : \"10.5.204.108\"\nexternal_ids        : {}\nlogical_port        : vpc1-external204\nmin_rx              : 100\nmin_tx              : 100\noptions             : {}\nstatus              : up\n\n### \u6ce8\u610f\u6240\u6709 status \u6b63\u5e38\u90fd\u5e94\u8be5\u662f up \u7684\n\n# 2. \u521b\u5efa\u4e86\u57fa\u4e8e bfd \u7684\u9759\u6001\u8def\u7531\n# k ko nbctl lr-route-list vpc1\nIPv4 Routes\nRoute Table &lt;main&gt;:\n           192.168.0.0/24              10.5.204.108 src-ip ecmp ecmp-symmetric-reply bfd\n           192.168.0.0/24              10.5.204.109 src-ip ecmp ecmp-symmetric-reply bfd\n           192.168.0.0/24              10.5.204.121 src-ip ecmp ecmp-symmetric-reply bfd\n\n# 3. \u9759\u6001\u8def\u7531\u8be6\u60c5\n# k ko nbctl find Logical_Router_Static_Route  policy=src-ip options=ecmp_symmetric_reply=\"true\"\n_uuid               : 3aacb384-d5ee-4b14-aebf-59e8c11717ba\nbfd                 : 684c4489-5b59-4693-8d8c-3beab93f8093\nexternal_ids        : {}\nip_prefix           : \"192.168.0.0/24\"\nnexthop             : \"10.5.204.109\"\noptions             : {ecmp_symmetric_reply=\"true\"}\noutput_port         : []\npolicy              : src-ip\nroute_table         : \"\"\n\n_uuid               : 18bcc585-bc05-430b-925b-ef673c8e1aef\nbfd                 : f0f62077-2ae9-4e79-b4f8-a446ec6e784c\nexternal_ids        : {}\nip_prefix           : \"192.168.0.0/24\"\nnexthop             : \"10.5.204.108\"\noptions             : {ecmp_symmetric_reply=\"true\"}\noutput_port         : []\npolicy              : src-ip\nroute_table         : \"\"\n\n_uuid               : 7d0a4e6b-cde0-4110-8176-fbaf19738498\nbfd                 : be7df545-2c4c-4751-878f-b3507987f050\nexternal_ids        : {}\nip_prefix           : \"192.168.0.0/24\"\nnexthop             : \"10.5.204.121\"\noptions             : {ecmp_symmetric_reply=\"true\"}\noutput_port         : []\npolicy              : src-ip\nroute_table         : \"\"\n</code></pre> <pre><code># \u540c\u65f6\u5728\u7f51\u5173\u8282\u70b9\u90fd\u5e94\u8be5\u5177\u5907\u4ee5\u4e0b\u8d44\u6e90\n\n[root@pc-node-1 ~]# ip netns exec ovnext bash ip a\n/usr/sbin/ip: /usr/sbin/ip: cannot execute binary file\n[root@pc-node-1 ~]#\n[root@pc-node-1 ~]# ip netns exec ovnext ip a\n1: lo: &lt;LOOPBACK,UP,LOWER_UP&gt; mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000\nlink/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00\n    inet 127.0.0.1/8 scope host lo\n       valid_lft forever preferred_lft forever\n    inet6 ::1/128 scope host\n       valid_lft forever preferred_lft forever\n1541: ovnext0: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1400 qdisc noqueue state UNKNOWN group default qlen 1000\nlink/ether 00:00:00:ab:bd:87 brd ff:ff:ff:ff:ff:ff\n    inet 10.5.204.108/24 brd 10.5.204.255 scope global ovnext0\n       valid_lft forever preferred_lft forever\n    inet6 fe80::200:ff:feab:bd87/64 scope link\n       valid_lft forever preferred_lft forever\n[root@pc-node-1 ~]#\n[root@pc-node-1 ~]# ip netns exec ovnext route -n\nKernel IP routing table\nDestination     Gateway         Genmask         Flags Metric Ref    Use Iface\n0.0.0.0         10.5.204.254    0.0.0.0         UG    0      0        0 ovnext0\n10.5.204.0      0.0.0.0         255.255.255.0   U     0      0        0 ovnext0\n\n## \u6ce8\u610f\u4ee5\u4e0a\u5185\u5bb9\u548c\u4e00\u4e2a internal port unerlay \u516c\u7f51 pod \u5185\u90e8\u7684 ns \u5927\u81f4\u662f\u4e00\u81f4\u7684\uff0c\u8fd9\u91cc\u53ea\u662f\u5728\u7f51\u5173\u8282\u70b9\u4e0a\u5355\u72ec\u7ef4\u62a4\u4e86\u4e00\u4e2a ns\n\n[root@pc-node-1 ~]# ip netns exec ovnext bfdd-control status\nThere are 1 sessions:\nSession 1\nid=1 local=10.5.204.108 (p) remote=10.5.204.122 state=Up\n\n## \u8fd9\u91cc\u5373\u662f lrp bfd \u4f1a\u8bdd\u7684\u53e6\u4e00\u7aef\uff0c\u4e5f\u662f lrp ecmp \u7684\u4e0b\u4e00\u8df3\u7684\u5176\u4e2d\u4e00\u4e2a\n\n\n[root@pc-node-1 ~]# ip netns exec ovnext ping -c1 223.5.5.5\nPING 223.5.5.5 (223.5.5.5) 56(84) bytes of data.\n64 bytes from 223.5.5.5: icmp_seq=1 ttl=115 time=21.6 ms\n\n# \u5230\u516c\u7f51\u6ca1\u95ee\u9898\n</code></pre> <p>\u53ef\u4ee5\u5728\u67d0\u4e00\u4e2a\u7f51\u5173\u8282\u70b9\u7684 ovnext ns \u5185\u6293\u5230\u51fa\u53bb\u7684\u5305</p> <pre><code># tcpdump -i ovnext0 host 223.5.5.5 -netvv\ndropped privs to tcpdump\ntcpdump: listening on ovnext0, link-type EN10MB (Ethernet), capture size 262144 bytes\n^C\n0 packets captured\n0 packets received by filter\n0 packets dropped by kernel\n[root@pc-node-1 ~]# exit\n[root@pc-node-1 ~]# ssh pc-node-2\nLast login: Thu Feb 23 09:21:08 2023 from 10.5.32.51\n[root@pc-node-2 ~]# ip netns exec ovnext bash\n[root@pc-node-2 ~]# tcpdump -i ovnext0 host 223.5.5.5 -netvv\ndropped privs to tcpdump\ntcpdump: listening on ovnext0, link-type EN10MB (Ethernet), capture size 262144 bytes\n^C\n0 packets captured\n0 packets received by filter\n0 packets dropped by kernel\n[root@pc-node-2 ~]# exit\n[root@pc-node-2 ~]# logout\nConnection to pc-node-2 closed.\n[root@pc-node-1 ~]# ssh pc-node-3\nLast login: Thu Feb 23 08:32:41 2023 from 10.5.32.51\n[root@pc-node-3 ~]#  ip netns exec ovnext bash\n[root@pc-node-3 ~]# tcpdump -i ovnext0 host 223.5.5.5 -netvv\ndropped privs to tcpdump\ntcpdump: listening on ovnext0, link-type EN10MB (Ethernet), capture size 262144 bytes\n00:00:00:2d:f8:ce &gt; 00:00:00:fd:b2:a4, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 57978, offset 0, flags [DF], proto ICMP (1), length 84)\n10.5.204.102 &gt; 223.5.5.5: ICMP echo request, id 22, seq 71, length 64\n00:00:00:fd:b2:a4 &gt; dc:ef:80:5a:44:1a, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 62, id 57978, offset 0, flags [DF], proto ICMP (1), length 84)\n10.5.204.102 &gt; 223.5.5.5: ICMP echo request, id 22, seq 71, length 64\n^C\n2 packets captured\n2 packets received by filter\n0 packets dropped by kernel\n[root@pc-node-3 ~]#\n\n# \u53ef\u4ee5\u5728\u8be5\u8282\u70b9 down \u6389\u51fa\u53bb\u7684\u7f51\u5361\uff0c\u7136\u540e\u770b pod \u51fa\u53bb\u7684\u5305\u5728\u7f51\u7edc\u4e2d\u65ad\u4e2d\u4f1a\u51fa\u73b0\u51e0\u4e2a\u5305\n# \u4e00\u822c\u90fd\u4f1a\u770b\u5230\u4e22 3 \u4e2a\u5305\n</code></pre>"},{"location":"advance/ovn-l3-ha-based-ecmp-with-bfd/#3-bfd","title":"3. \u5173\u95ed bfd \u6a21\u5f0f","text":"<p>\u5728\u67d0\u4e9b\u573a\u666f\u4e0b\uff0c\u53ef\u80fd\u60f3\u76f4\u63a5\u4f7f\u7528\uff08\u96c6\u4e2d\u5f0f\uff09\u5355\u4e2a\u7f51\u5173\u76f4\u63a5\u51fa\u516c\u7f51\uff0c\u8fd9\u4e2a\u65f6\u5019\u548c\u9ed8\u8ba4 vpc enable_eip_snat \u7684\u4f7f\u7528\u6a21\u5f0f\u662f\u4e00\u81f4\u7684</p> <pre><code># cat 01-vpc-ecmp-enable-external-bfd.yml\nkind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\n  name: vpc2\nspec:\n  namespaces:\n  - vpc2\n  enableExternal: true\n#enableBfd: true\nenableBfd: false\n\n## \u5c06 bfd \u529f\u80fd\u76f4\u63a5\u7981\u7528\u5373\u53ef\n\n# k ko nbctl lr-route-list vpc2\nIPv4 Routes\nRoute Table &lt;main&gt;:\n                0.0.0.0/0              10.5.204.254 dst-ip\n\n# \u5e94\u7528\u540e\u8def\u7531\u4f1a\u5207\u6362\u56de\u6b63\u5e38\u7684\u9ed8\u8ba4\u9759\u6001\u8def\u7531\n# \u540c\u65f6 nbctl list bfd  \u53ef\u4ee5\u770b\u5230 lrp \u5173\u8054\u7684 bfd \u4f1a\u8bdd\u5df2\u7ecf\u79fb\u9664\n# \u800c\u4e14 ovnext ns \u4e2d\u7684\u5bf9\u7aef bfd \u4f1a\u8bdd\u4e5f\u81ea\u52a8\u79fb\u9664\n# \u8be5\u5207\u6362\u8fc7\u7a0b\u4fdd\u6301 vpc subnet \u5185\u4fdd\u6301 ping \u672a\u770b\u5230(\u79d2\u7ea7)\u4e22\u5305\n# \u518d\u5207\u6362\u56de\u53bb \u4e5f\u672a\u770b\u5230(\u79d2\u7ea7)\u4e22\u5305\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/ovn-remote-port-mirroring/","title":"OVN \u6d41\u91cf\u955c\u50cf","text":"<p>\u6b64\u529f\u80fd\u53ef\u4ee5\u5c06\u6307\u5b9a Pod\u3001\u6307\u5b9a\u65b9\u5411\u7684\u6d41\u91cf\uff0c\u901a\u8fc7 GRE/ERSPAN \u5c01\u88c5\u540e\uff0c\u4f20\u8f93\u5230\u8fdc\u7aef\u3002</p> <p>\u6b64\u529f\u80fd\u8981\u6c42 Kube-OVN \u7248\u672c\u4e0d\u4f4e\u4e8e v1.12\u3002</p>"},{"location":"advance/ovn-remote-port-mirroring/#multus-cni","title":"\u90e8\u7f72 Multus-CNI","text":"<p>\u53c2\u8003 Multus-CNI \u6587\u6863 \u90e8\u7f72 Multus\u3002</p>"},{"location":"advance/ovn-remote-port-mirroring/#_1","title":"\u521b\u5efa\u9644\u5c5e\u7f51\u7edc","text":"<p>\u4f7f\u7528\u4ee5\u4e0b\u5185\u5bb9\u521b\u5efa\u9644\u5c5e\u7f51\u7edc\uff1a</p> <pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: attachnet\nnamespace: default\nspec:\nconfig: |\n{\n\"cniVersion\": \"0.3.1\",\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"attachnet.default.ovn\"\n}\n</code></pre> <p>\u5176\u4e2d <code>provider</code> \u5b57\u6bb5\u683c\u5f0f\u4e3a <code>&lt;NAME&gt;.&lt;NAMESPACE&gt;.ovn</code>\u3002</p>"},{"location":"advance/ovn-remote-port-mirroring/#underlay","title":"\u521b\u5efa Underlay \u7f51\u7edc","text":"<p>\u955c\u50cf\u6d41\u91cf\u662f\u5c01\u88c5\u540e\u8fdb\u884c\u4f20\u8f93\u7684\uff0c\u56e0\u6b64\u7528\u4e8e\u4f20\u8f93\u7684\u7f51\u7edc\uff0cMTU \u9700\u8981\u5927\u4e8e\u88ab\u955c\u50cf\u7684 LSP/Pod\u3002\u8fd9\u91cc\u6211\u4eec\u4f7f\u7528 Underlay \u4f5c\u4e3a\u4f20\u8f93\u7f51\u7edc\u3002</p> <p>\u4f7f\u7528\u4ee5\u4e0b\u5185\u5bb9\u521b\u5efa Underlay \u7f51\u7edc\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: ProviderNetwork\nmetadata:\nname: net1\nspec:\ndefaultInterface: eth1\n---\napiVersion: kubeovn.io/v1\nkind: Vlan\nmetadata:\nname: vlan1\nspec:\nid: 0\nprovider: net1\n---\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: subnet1\nspec:\nprotocol: IPv4\ncidrBlock: 172.19.0.0/16\nexcludeIps:\n- 172.19.0.2..172.19.0.20\ngateway: 172.19.0.1\nvlan: vlan1\nprovider: attachnet.default.ovn\n</code></pre> <p>\u5176\u4e2d\uff0c\u5b50\u7f51\u7684 <code>provider</code> \u5fc5\u987b\u4e0e\u9644\u5c5e\u7f51\u7edc\u7684 <code>provider</code> \u76f8\u540c\u3002</p>"},{"location":"advance/ovn-remote-port-mirroring/#pod","title":"\u521b\u5efa\u6d41\u91cf\u63a5\u6536 Pod","text":"<p>\u4f7f\u7528\u4ee5\u4e0b\u5185\u5bb9\u521b\u5efa\u7528\u4e8e\u63a5\u6536\u955c\u50cf\u6d41\u91cf\u7684 Pod\uff1a</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: pod1\nannotations:\nk8s.v1.cni.cncf.io/networks: default/attachnet\nspec:\ncontainers:\n- name: bash\nimage: docker.io/kubeovn/kube-ovn:v1.13.0\nargs:\n- bash\n- -c\n- sleep infinity\nsecurityContext:\nprivileged: true\n</code></pre> <p>\u521b\u5efa\u5b8c\u6210\u540e\uff0c\u67e5\u770b Pod \u7684 IP \u5730\u5740\uff1a</p> <pre><code>$ kubectl get ips | grep pod1\npod1.default                        10.16.0.12   00:00:00:FF:34:24  kube-ovn-worker  ovn-default\npod1.default.attachnet.default.ovn  172.19.0.21  00:00:00:A0:30:68  kube-ovn-worker  subnet1\n</code></pre> <p>\u8bb0\u4f4f\u7b2c\u4e8c\u7f51\u5361\u7684 IP \u5730\u5740 <code>172.19.0.21</code>\u3002</p>"},{"location":"advance/ovn-remote-port-mirroring/#ovn_1","title":"\u521b\u5efa OVN \u6d41\u91cf\u955c\u50cf","text":"<p>\u4f7f\u7528\u4ee5\u4e0b\u547d\u4ee4\u521b\u5efa OVN \u6d41\u91cf\u955c\u50cf\uff1a</p> <pre><code>kubectl ko nbctl mirror-add mirror1 gre 99 from-lport 172.19.0.21\nkubectl ko nbctl lsp-attach-mirror coredns-787d4945fb-gpnkb.kube-system mirror1\n</code></pre> <p>\u5176\u4e2d <code>coredns-787d4945fb-gpnkb.kube-system</code> \u662f OVN LSP \u7684\u540d\u79f0\uff0c\u683c\u5f0f\u901a\u5e38\u4e3a <code>&lt;POD_NAME&gt;.&lt;POD_NAMESPACE&gt;</code>\u3002</p> <p>\u76f8\u5173\u7684 OVN \u547d\u4ee4\u4f7f\u7528\u65b9\u6cd5\u5982\u4e0b\uff1a</p> <pre><code>ovn-nbctl mirror-add &lt;NAME&gt; &lt;TYPE&gt; &lt;INDEX&gt; &lt;FILTER&gt; &lt;IP&gt;\n\nNAME   - add a mirror with given name\nTYPE   - specify TYPE 'gre' or 'erspan'\nINDEX  - specify the tunnel INDEX value\n         (indicates key if GRE, erpsan_idx if ERSPAN)\nFILTER - specify FILTER for mirroring selection\n         ('to-lport' / 'from-lport')\nIP     - specify Sink / Destination i.e. Remote IP\n\novn-nbctl mirror-del [NAME]         remove mirrors\novn-nbctl mirror-list               print mirrors\n\novn-nbctl lsp-attach-mirror PORT MIRROR   attach source PORT to MIRROR\novn-nbctl lsp-detach-mirror PORT MIRROR   detach source PORT from MIRROR\n</code></pre>"},{"location":"advance/ovn-remote-port-mirroring/#pod_1","title":"\u914d\u7f6e\u6d41\u91cf\u63a5\u6536 Pod","text":"<p>\u5728\u524d\u9762\u521b\u5efa\u7684 Pod \u4e2d\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff1a</p> <pre><code>root@pod1:/kube-ovn# ip link add mirror1 type gretap local 172.19.0.21 key 99 dev net1\nroot@pod1:/kube-ovn# ip link set mirror1 up\n</code></pre> <p>\u63a5\u4e0b\u6765\u5c31\u53ef\u4ee5\u5728\u63a5\u6536\u6d41\u91cf\u7684 Pod \u4e2d\u8fdb\u884c\u6293\u5305\u9a8c\u8bc1\uff1a</p> <pre><code>root@pod1:/kube-ovn# tcpdump -i mirror1 -nnve\ntcpdump: listening on mirror1, link-type EN10MB (Ethernet), snapshot length 262144 bytes\n05:13:30.328808 00:00:00:a3:f5:e2 &gt; 00:00:00:97:0f:6e, ethertype ARP (0x0806), length 42: Ethernet (len 6), IPv4 (len 4), Request who-has 10.16.0.7 tell 10.16.0.4, length 28\n05:13:30.559167 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 212: (tos 0x0, ttl 64, id 57364, offset 0, flags [DF], proto UDP (17), length 198)\n10.16.0.4.53 &gt; 10.16.0.6.50472: 34511 NXDomain*- 0/1/1 (170)\n05:13:30.559343 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 212: (tos 0x0, ttl 64, id 57365, offset 0, flags [DF], proto UDP (17), length 198)\n10.16.0.4.53 &gt; 10.16.0.6.45177: 1659 NXDomain*- 0/1/1 (170)\n05:13:30.560625 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 200: (tos 0x0, ttl 64, id 57367, offset 0, flags [DF], proto UDP (17), length 186)\n10.16.0.4.53 &gt; 10.16.0.6.43848: 2636*- 0/1/1 (158)\n05:13:30.562774 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 191: (tos 0x0, ttl 64, id 57368, offset 0, flags [DF], proto UDP (17), length 177)\n10.16.0.4.53 &gt; 10.16.0.6.37755: 48737 NXDomain*- 0/1/1 (149)\n05:13:30.563523 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 187: (tos 0x0, ttl 64, id 57369, offset 0, flags [DF], proto UDP (17), length 173)\n10.16.0.4.53 &gt; 10.16.0.6.53887: 45519 NXDomain*- 0/1/1 (145)\n05:13:30.564940 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 201: (tos 0x0, ttl 64, id 57370, offset 0, flags [DF], proto UDP (17), length 187)\n10.16.0.4.53 &gt; 10.16.0.6.40846: 25745 NXDomain*- 0/1/1 (159)\n05:13:30.565140 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 201: (tos 0x0, ttl 64, id 57371, offset 0, flags [DF], proto UDP (17), length 187)\n10.16.0.4.53 &gt; 10.16.0.6.45214: 61875 NXDomain*- 0/1/1 (159)\n05:13:30.566023 00:00:00:a3:f5:e2 &gt; 00:00:00:55:e4:4e, ethertype IPv4 (0x0800), length 80: (tos 0x0, ttl 64, id 45937, offset 0, flags [DF], proto UDP (17), length 66)\n10.16.0.4.44116 &gt; 172.18.0.1.53: 16025+ [1au] AAAA? alauda.cn. (38)\n</code></pre>"},{"location":"advance/ovn-remote-port-mirroring/#_2","title":"\u6ce8\u610f\u4e8b\u9879","text":"<ol> <li>\u5982\u679c\u4f7f\u7528 ERSPAN \u4f5c\u4e3a\u5c01\u88c5\u534f\u8bae\uff0cOVN \u8282\u70b9\u53ca\u8fdc\u7aef\u8bbe\u5907\u7684 Linux \u5185\u6838\u7248\u672c\u4e0d\u5f97\u4f4e\u4e8e 4.14\u3002\u82e5\u4f7f\u7528 ERSPAN \u4f5c\u4e3a\u5c01\u88c5\u534f\u8bae\u4e14\u4f7f\u7528 IPv6 \u4f5c\u4e3a\u4f20\u8f93\u7f51\u7edc\uff0cLinux \u5185\u6838\u7248\u672c\u4e0d\u5f97\u4f4e\u4e8e 4.16\u3002</li> <li>\u955c\u50cf\u6d41\u91cf\u7684\u4f20\u8f93\u662f\u5355\u5411\u7684\uff0c\u53ea\u9700\u4fdd\u8bc1 OVN \u8282\u70b9\u80fd\u591f\u8bbf\u95ee\u8fdc\u7aef\u8bbe\u5907\u5373\u53ef\u3002</li> </ol> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/performance-tuning/","title":"\u6027\u80fd\u8c03\u4f18","text":"<p>\u4e3a\u4e86\u4fdd\u6301\u5b89\u88c5\u7684\u7b80\u5355\u548c\u529f\u80fd\u7684\u5b8c\u5907\uff0cKube-OVN \u7684\u9ed8\u8ba4\u5b89\u88c5\u811a\u672c\u5e76\u6ca1\u6709\u5bf9\u6027\u80fd\u9488\u5bf9\u6027\u7684\u4f18\u5316\u3002\u5982\u679c\u5e94\u7528\u5bf9\u5ef6\u8fdf\u548c\u541e\u5410\u91cf\u654f\u611f\uff0c \u7ba1\u7406\u5458\u53ef\u4ee5\u901a\u8fc7\u672c\u6587\u6863\u5bf9\u6027\u80fd\u8fdb\u884c\u9488\u5bf9\u6027\u4f18\u5316\u3002</p> <p>\u793e\u533a\u4f1a\u4e0d\u65ad\u8fed\u4ee3\u63a7\u5236\u9762\u677f\u548c\u4f18\u5316\u9762\u7684\u6027\u80fd\uff0c\u90e8\u5206\u901a\u7528\u6027\u80fd\u4f18\u5316\u5df2\u7ecf\u96c6\u6210\u5230\u6700\u65b0\u7248\u672c\uff0c\u5efa\u8bae\u4f7f\u7528\u6700\u65b0\u7248\u672c\u83b7\u5f97\u66f4\u597d\u7684\u9ed8\u8ba4\u6027\u80fd\u3002</p> <p>\u66f4\u591a\u5173\u4e8e\u6027\u80fd\u4f18\u5316\u7684\u8fc7\u7a0b\u548c\u65b9\u6cd5\u8bba\uff0c\u53ef\u4ee5\u89c2\u770b\u89c6\u9891\u5206\u4eab\uff1aKube-OVN \u5bb9\u5668\u6027\u80fd\u4f18\u5316\u4e4b\u65c5\u3002</p>"},{"location":"advance/performance-tuning/#_2","title":"\u57fa\u51c6\u6d4b\u8bd5","text":"<p>\u7531\u4e8e\u8f6f\u786c\u4ef6\u73af\u5883\u7684\u5dee\u5f02\u6781\u5927\uff0c\u8fd9\u91cc\u63d0\u4f9b\u7684\u6027\u80fd\u6d4b\u8bd5\u6570\u636e\u53ea\u80fd\u4f5c\u4e3a\u53c2\u8003\uff0c\u5b9e\u9645\u6d4b\u8bd5\u7ed3\u679c\u4f1a\u548c\u672c\u6587\u6863\u4e2d\u7684\u7ed3\u679c\u5b58\u5728\u8f83\u5927\u5dee\u5f02\u3002 \u5efa\u8bae\u6bd4\u8f83\u4f18\u5316\u524d\u540e\u7684\u6027\u80fd\u6d4b\u8bd5\u7ed3\u679c\uff0c\u548c\u5bbf\u4e3b\u673a\u7f51\u7edc\u548c\u5bb9\u5668\u7f51\u7edc\u7684\u6027\u80fd\u6bd4\u8f83\u3002</p>"},{"location":"advance/performance-tuning/#overlay","title":"Overlay \u4f18\u5316\u524d\u540e\u6027\u80fd\u5bf9\u6bd4","text":"<p>\u73af\u5883\u4fe1\u606f\uff1a</p> <ul> <li>Kubernetes: 1.22.0</li> <li>OS: CentOS 7</li> <li>Kube-OVN: 1.8.0 Overlay \u6a21\u5f0f</li> <li>CPU: Intel(R) Xeon(R) E-2278G</li> <li>Network: 2*10Gbps, xmit_hash_policy=layer3+4</li> </ul> <p>\u6211\u4eec\u4f7f\u7528 <code>qperf -t 60 &lt;server ip&gt; -ub -oo msg_size:1 -vu tcp_lat tcp_bw udp_lat udp_bw</code> \u6d4b\u8bd5  1 \u5b57\u8282\u5c0f\u5305\u4e0b tcp/udp \u7684\u5e26\u5bbd\u548c\u5ef6\u8fdf\uff0c\u5206\u522b\u6d4b\u8bd5\u4f18\u5316\u524d\uff0c\u4f18\u5316\u540e\u4ee5\u53ca\u5bbf\u4e3b\u673a\u7f51\u7edc\u7684\u6027\u80fd\uff1a</p> Type tcp_lat (us) udp_lat (us) tcp_bw (Mb/s) udp_bw(Mb/s) Kube-OVN Default 25.7 22.9 27.1 1.59 Kube-OVN Optimized 13.9 12.9 27.6 5.57 HOST Network 13.1 12.4 28.2 6.02"},{"location":"advance/performance-tuning/#overlay-underlay-calico","title":"Overlay\uff0c Underlay \u4ee5\u53ca Calico \u4e0d\u540c\u6a21\u5f0f\u6027\u80fd\u5bf9\u6bd4","text":"<p>\u4e0b\u9762\u6211\u4eec\u4f1a\u6bd4\u8f83\u4f18\u5316\u540e Kube-OVN \u5728\u4e0d\u540c\u5305\u5927\u5c0f\u4e0b\u7684 Overlay \u548c Underlay \u6027\u80fd\uff0c\u5e76\u548c Calico \u7684 <code>IPIP Always</code>, <code>IPIP never</code> \u4ee5\u53ca\u5bbf\u4e3b\u673a\u7f51\u7edc\u505a\u6bd4\u8f83\u3002</p> <p>Environment:</p> <ul> <li>Kubernetes: 1.22.0</li> <li>OS: CentOS 7</li> <li>Kube-OVN: 1.8.0</li> <li>CPU: AMD EPYC 7402P 24-Core Processor</li> <li>Network: Intel Corporation Ethernet Controller XXV710 for 25GbE SFP28</li> </ul> <p><code>qperf -t 60 &lt;server ip&gt; -ub -oo msg_size:1 -vu tcp_lat tcp_bw udp_lat udp_bw</code></p> Type tcp_lat (us) udp_lat (us) tcp_bw (Mb/s) udp_bw(Mb/s) Kube-OVN Overlay 15.2 14.6 23.6 2.65 Kube-OVN Underlay 14.3 13.8 24.2 3.46 Calico IPIP 21.4 20.2 23.6 1.18 Calico NoEncap 19.3 16.9 23.6 1.76 HOST Network 16.6 15.4 24.8 2.64 <p><code>qperf -t 60 &lt;server ip&gt; -ub -oo msg_size:1K -vu tcp_lat tcp_bw udp_lat udp_bw</code></p> Type tcp_lat (us) udp_lat (us) tcp_bw (Gb/s) udp_bw(Gb/s) Kube-OVN Overlay 16.5 15.8 10.2 2.77 Kube-OVN Underlay 15.9 14.5 9.6 3.22 Calico IPIP 22.5 21.5 1.45 1.14 Calico NoEncap 19.4 18.3 3.76 1.63 HOST Network 18.1 16.6 9.32 2.66 <p><code>qperf -t 60 &lt;server ip&gt; -ub -oo msg_size:4K -vu tcp_lat tcp_bw udp_lat udp_bw</code></p> Type tcp_lat (us) udp_lat (us) tcp_bw (Gb/s) udp_bw(Gb/s) Kube-OVN Overlay 34.7 41.6 16.0 9.23 Kube-OVN Underlay 32.6 44 15.1 6.71 Calico IPIP 44.8 52.9 2.94 3.26 Calico NoEncap 40 49.6 6.56 4.19 HOST Network 35.9 45.9 14.6 5.59 <p>\u5728\u90e8\u5206\u60c5\u51b5\u4e0b\u5bb9\u5668\u7f51\u7edc\u7684\u6027\u80fd\u4f1a\u4f18\u4e8e\u5bbf\u4e3b\u673a\u7f51\u7edc\uff0c\u8fd9\u662f\u4f18\u4e8e\u7ecf\u8fc7\u4f18\u5316\u540e\u5bb9\u5668\u7f51\u7edc\u8def\u5f84\u5b8c\u5168\u7ed5\u8fc7\u4e86 netfilter\uff0c \u800c\u5bbf\u4e3b\u673a\u7f51\u7edc\u7531\u4e8e <code>kube-proxy</code> \u7684\u5b58\u5728\u6240\u6709\u6570\u636e\u5305\u5747\u9700\u7ecf\u8fc7 netfilter\uff0c\u4f1a\u5bfc\u81f4\u5728\u4e00\u4e9b\u73af\u5883\u4e0b\u5bb9\u5668\u7f51\u7edc \u7684\u6d88\u8017\u76f8\u5bf9\u66f4\u5c0f\uff0c\u56e0\u6b64\u4f1a\u6709\u66f4\u597d\u7684\u6027\u80fd\u8868\u73b0\u3002</p>"},{"location":"advance/performance-tuning/#_3","title":"\u6570\u636e\u5e73\u9762\u6027\u80fd\u4f18\u5316\u65b9\u6cd5","text":"<p>\u8fd9\u91cc\u4ecb\u7ecd\u7684\u4f18\u5316\u65b9\u6cd5\u548c\u8f6f\u786c\u4ef6\u73af\u5883\u4ee5\u53ca\u6240\u9700\u8981\u7684\u529f\u80fd\u76f8\u5173\uff0c\u8bf7\u4ed4\u7ec6\u4e86\u89e3\u4f18\u5316\u7684\u524d\u63d0\u6761\u4ef6\u518d\u8fdb\u884c\u5c1d\u8bd5\u3002</p>"},{"location":"advance/performance-tuning/#cpu","title":"CPU \u6027\u80fd\u6a21\u5f0f\u8c03\u6574","text":"<p>\u90e8\u5206\u73af\u5883\u4e0b CPU \u8fd0\u884c\u5728\u8282\u80fd\u6a21\u5f0f\uff0c\u8be5\u6a21\u5f0f\u4e0b\u6027\u80fd\u8868\u73b0\u5c06\u4f1a\u4e0d\u7a33\u5b9a\uff0c\u5ef6\u8fdf\u4f1a\u51fa\u73b0\u660e\u663e\u589e\u52a0\uff0c\u5efa\u8bae\u4f7f\u7528 CPU \u7684\u6027\u80fd\u6a21\u5f0f\u83b7\u5f97\u66f4\u7a33\u5b9a\u7684\u6027\u80fd\u8868\u73b0\uff1a</p> <pre><code>cpupower frequency-set -g performance\n</code></pre>"},{"location":"advance/performance-tuning/#_4","title":"\u7f51\u5361\u786c\u4ef6\u961f\u5217\u8c03\u6574","text":"<p>\u5728\u6d41\u91cf\u589e\u5927\u7684\u60c5\u51b5\u4e0b\uff0c\u7f13\u51b2\u961f\u5217\u8fc7\u77ed\u53ef\u80fd\u5bfc\u81f4\u8f83\u9ad8\u7684\u4e22\u5305\u7387\u5bfc\u81f4\u6027\u80fd\u663e\u8457\u4e0b\u964d\uff0c\u9700\u8981\u8fdb\u884c\u8c03\u6574</p> <p>\u68c0\u67e5\u5f53\u524d\u7f51\u5361\u961f\u5217\u957f\u5ea6\uff1a</p> <pre><code># ethtool -g eno1\nRing parameters for eno1:\n Pre-set maximums:\n RX:             4096\nRX Mini:        0\nRX Jumbo:       0\nTX:             4096\nCurrent hardware settings:\n RX:             255\nRX Mini:        0\nRX Jumbo:       0\nTX:             255\n</code></pre> <p>\u589e\u52a0\u961f\u5217\u957f\u5ea6\u81f3\u6700\u5927\u503c\uff1a</p> <pre><code>ethtool -G eno1 rx 4096\nethtool -G eno1 tx 4096\n</code></pre>"},{"location":"advance/performance-tuning/#tuned","title":"\u4f7f\u7528 tuned \u4f18\u5316\u7cfb\u7edf\u53c2\u6570","text":"<p>tuned \u53ef\u4ee5\u4f7f\u7528\u4e00\u7cfb\u5217\u9884\u7f6e\u7684 profile \u6587\u4ef6\u4fdd\u5b58\u4e86\u9488\u5bf9\u7279\u5b9a\u573a\u666f\u7684\u4e00\u7cfb\u5217\u7cfb\u7edf\u4f18\u5316\u914d\u7f6e\u3002</p> <p>\u9488\u5bf9\u5ef6\u8fdf\u4f18\u5148\u573a\u666f\uff1a</p> <pre><code>tuned-adm profile network-latency\n</code></pre> <p>\u9488\u5bf9\u541e\u5410\u91cf\u4f18\u5148\u573a\u666f\uff1a</p> <pre><code>tuned-adm profile network-throughput\n</code></pre>"},{"location":"advance/performance-tuning/#_5","title":"\u4e2d\u65ad\u7ed1\u5b9a","text":"<p>\u6211\u4eec\u63a8\u8350\u7981\u7528 <code>irqbalance</code> \u5e76\u5c06\u7f51\u5361\u4e2d\u65ad\u548c\u7279\u5b9a CPU \u8fdb\u884c\u7ed1\u5b9a\uff0c\u6765\u907f\u514d\u5728\u591a\u4e2a CPU \u4e4b\u95f4\u5207\u6362\u5bfc\u81f4\u7684\u6027\u80fd\u6ce2\u52a8\u3002</p>"},{"location":"advance/performance-tuning/#ovn-lb","title":"\u5173\u95ed OVN LB","text":"<p>OVN \u7684 L2 LB \u5b9e\u73b0\u8fc7\u7a0b\u4e2d\u9700\u8981\u8c03\u7528\u5185\u6838\u7684 <code>conntrack</code> \u6a21\u5757\u5e76\u8fdb\u884c recirculate \u5bfc\u81f4\u5927\u91cf\u7684 CPU \u5f00\u9500\uff0c\u7ecf\u6d4b\u8bd5\u8be5\u529f\u80fd\u4f1a\u5e26\u6765 20% \u5de6\u53f3\u7684 CPU \u5f00\u9500\uff0c \u5728 Overlay \u7f51\u7edc\u6a21\u5f0f\u4e0b\u53ef\u4ee5\u4f7f\u7528 <code>kube-proxy</code> \u5b8c\u6210 Service \u8f6c\u53d1\u529f\u80fd\uff0c\u83b7\u5f97\u66f4\u597d\u7684 Pod-to-Pod \u6027\u80fd\u3002\u53ef\u4ee5\u5728 <code>kube-ovn-controller</code> \u4e2d\u5173\u95ed\u8be5\u529f\u80fd\uff1a</p> <pre><code>command:\n- /kube-ovn/start-controller.sh\nargs:\n...\n- --enable-lb=false\n...\n</code></pre> <p>Underlay \u6a21\u5f0f\u4e0b <code>kube-proxy</code> \u65e0\u6cd5\u4f7f\u7528 iptables \u6216 ipvs \u63a7\u5236\u5bb9\u5668\u7f51\u7edc\u6d41\u91cf\uff0c\u5982\u9700\u5173\u95ed LB \u529f\u80fd\u9700\u8981\u786e\u8ba4\u662f\u5426\u4e0d\u9700\u8981 Service \u529f\u80fd\u3002</p>"},{"location":"advance/performance-tuning/#fastpath","title":"\u5185\u6838 FastPath \u6a21\u5757","text":"<p>\u7531\u4e8e\u5bb9\u5668\u7f51\u7edc\u548c\u5bbf\u4e3b\u673a\u7f51\u7edc\u5728\u4e0d\u540c\u7684 network ns\uff0c\u6570\u636e\u5305\u5728\u8de8\u5bbf\u4e3b\u673a\u4f20\u8f93\u65f6\u4f1a\u591a\u6b21\u7ecf\u8fc7 netfilter \u6a21\u5757\uff0c\u4f1a\u5e26\u6765\u8fd1 20% \u7684 CPU \u5f00\u9500\u3002\u7531\u4e8e\u5927\u90e8\u5206\u60c5\u51b5\u4e0b \u5bb9\u5668\u7f51\u7edc\u5185\u5e94\u7528\u65e0\u987b\u4f7f\u7528 netfilter \u6a21\u5757\u7684\u529f\u80fd\uff0c<code>FastPath</code> \u6a21\u5757\u53ef\u4ee5\u7ed5\u8fc7 netfilter \u964d\u4f4e CPU \u5f00\u9500\u3002</p> <p>\u5982\u5bb9\u5668\u7f51\u7edc\u5185\u9700\u8981\u4f7f\u7528 netfilter \u63d0\u4f9b\u7684\u529f\u80fd\u5982 iptables\uff0cipvs\uff0cnftables \u7b49\uff0c\u8be5\u6a21\u5757\u4f1a\u4f7f\u76f8\u5173\u529f\u80fd\u5931\u6548\u3002</p> <p>\u7531\u4e8e\u5185\u6838\u6a21\u5757\u548c\u5185\u6838\u7248\u672c\u76f8\u5173\uff0c\u65e0\u6cd5\u63d0\u4f9b\u4e00\u4e2a\u5355\u4e00\u9002\u5e94\u6240\u6709\u5185\u6838\u7684\u5185\u6838\u6a21\u5757\u5236\u54c1\u3002\u6211\u4eec\u9884\u5148\u7f16\u8bd1\u4e86\u90e8\u5206\u5185\u6838\u7684 <code>FastPath</code> \u6a21\u5757\uff0c \u53ef\u4ee5\u524d\u5f80 tunning-package \u8fdb\u884c\u4e0b\u8f7d\u3002</p> <p>\u4e5f\u53ef\u4ee5\u624b\u52a8\u8fdb\u884c\u7f16\u8bd1\uff0c\u65b9\u6cd5\u53c2\u8003\u624b\u52a8\u7f16\u8bd1 FastPath \u6a21\u5757</p> <p>\u83b7\u5f97\u5185\u6838\u6a21\u5757\u540e\u53ef\u5728\u6bcf\u4e2a\u8282\u70b9\u4f7f\u7528 <code>insmod kube_ovn_fastpath.ko</code> \u52a0\u8f7d <code>FastPath</code> \u6a21\u5757\uff0c\u5e76\u4f7f\u7528 <code>dmesg</code> \u9a8c\u8bc1\u6a21\u5757\u52a0\u8f7d\u6210\u529f\uff1a</p> <pre><code># dmesg\n...\n[619631.323788] init_module,kube_ovn_fastpath_local_out\n[619631.323798] init_module,kube_ovn_fastpath_post_routing\n[619631.323800] init_module,kube_ovn_fastpath_pre_routing\n[619631.323801] init_module,kube_ovn_fastpath_local_in\n...\n</code></pre>"},{"location":"advance/performance-tuning/#ovs","title":"OVS \u5185\u6838\u6a21\u5757\u4f18\u5316","text":"<p>OVS \u7684 flow \u5904\u7406\u5305\u62ec\u54c8\u5e0c\u8ba1\u7b97\uff0c\u5339\u914d\u7b49\u64cd\u4f5c\u4f1a\u6d88\u8017\u5927\u7ea6 10% \u5de6\u53f3\u7684 CPU \u8d44\u6e90\u3002\u73b0\u4ee3 x86 CPU \u4e0a\u7684\u4e00\u4e9b\u6307\u4ee4\u96c6\u4f8b\u5982 <code>popcnt</code> \u548c <code>sse4.2</code> \u53ef\u4ee5 \u52a0\u901f\u76f8\u5173\u8ba1\u7b97\u8fc7\u7a0b\uff0c\u4f46\u5185\u6838\u9ed8\u8ba4\u7f16\u8bd1\u672a\u5f00\u542f\u76f8\u5173\u9009\u9879\u3002\u7ecf\u6d4b\u8bd5\u5728\u5f00\u542f\u76f8\u5e94\u6307\u4ee4\u96c6\u4f18\u5316\u540e\uff0cflow \u76f8\u5173\u64cd\u4f5c CPU \u6d88\u8017\u5c06\u4f1a\u964d\u81f3 5% \u5de6\u53f3\u3002</p> <p>\u548c <code>FastPath</code> \u6a21\u5757\u7684\u7f16\u8bd1\u7c7b\u4f3c\uff0c\u7531\u4e8e\u5185\u6838\u6a21\u5757\u548c\u5185\u6838\u7248\u672c\u76f8\u5173\uff0c\u65e0\u6cd5\u63d0\u4f9b\u4e00\u4e2a\u5355\u4e00\u9002\u5e94\u6240\u6709\u5185\u6838\u7684\u5185\u6838\u6a21\u5757\u5236\u54c1\u3002\u7528\u6237\u9700\u8981\u624b\u52a8\u7f16\u8bd1\u6216\u8005 \u524d\u5f80 tunning-package \u67e5\u770b\u662f\u5426\u6709\u5df2\u7f16\u8bd1\u597d\u7684\u5236\u54c1\u8fdb\u884c\u4e0b\u8f7d\u3002</p> <p>\u4f7f\u7528\u8be5\u5185\u6838\u6a21\u5757\u524d\u8bf7\u5148\u786e\u8ba4 CPU \u662f\u5426\u652f\u6301\u76f8\u5173\u6307\u4ee4\u96c6\uff1a</p> <pre><code>cat /proc/cpuinfo  | grep popcnt\ncat /proc/cpuinfo  | grep sse4_2\n</code></pre>"},{"location":"advance/performance-tuning/#centos","title":"CentOS \u4e0b\u7f16\u8bd1\u5b89\u88c5","text":"<p>\u5b89\u88c5\u76f8\u5173\u7f16\u8bd1\u4f9d\u8d56\u548c\u5185\u6838\u5934\u6587\u4ef6\uff1a</p> <pre><code>yum install -y gcc kernel-devel-$(uname -r) python3 autoconf automake libtool rpm-build openssl-devel\n</code></pre> <p>\u7f16\u8bd1 OVS \u5185\u6838\u6a21\u5757\u5e76\u751f\u6210\u5bf9\u5e94 RPM \u6587\u4ef6:</p> <pre><code>git clone -b branch-2.17 --depth=1 https://github.com/openvswitch/ovs.git\ncd ovs\ncurl -s  https://github.com/kubeovn/ovs/commit/2d2c83c26d4217446918f39d5cd5838e9ac27b32.patch |  git apply\n./boot.sh\n./configure --with-linux=/lib/modules/$(uname -r)/build CFLAGS=\"-g -O2 -mpopcnt -msse4.2\"\nmake rpm-fedora-kmod\ncd rpm/rpmbuild/RPMS/x86_64/\n</code></pre> <p>\u590d\u5236 RPM \u5230\u6bcf\u4e2a\u8282\u70b9\u5e76\u8fdb\u884c\u5b89\u88c5\uff1a</p> <pre><code>rpm -i openvswitch-kmod-2.15.2-1.el7.x86_64.rpm\n</code></pre> <p>\u82e5\u4e4b\u524d\u5df2\u7ecf\u542f\u52a8\u8fc7 Kube-OVN\uff0c\u65e7\u7248\u672c OVS \u6a21\u5757\u5df2\u52a0\u8f7d\u81f3\u5185\u6838\uff0c\u5efa\u8bae\u91cd\u542f\u673a\u5668\u91cd\u65b0\u52a0\u8f7d\u65b0\u7248\u5185\u6838\u6a21\u5757\u3002</p>"},{"location":"advance/performance-tuning/#ubuntu","title":"Ubuntu \u4e0b\u7f16\u8bd1\u5b89\u88c5","text":"<p>\u5b89\u88c5\u76f8\u5173\u7f16\u8bd1\u4f9d\u8d56\u548c\u5185\u6838\u5934\u6587\u4ef6\uff1a</p> <pre><code>apt install -y autoconf automake libtool gcc build-essential libssl-dev\n</code></pre> <p>\u7f16\u8bd1 OVS \u5185\u6838\u6a21\u5757\u5e76\u5b89\u88c5\uff1a</p> <pre><code>apt install -y autoconf automake libtool gcc build-essential libssl-dev\n\ngit clone -b branch-2.17 --depth=1 https://github.com/openvswitch/ovs.git\ncd ovs\ncurl -s  https://github.com/kubeovn/ovs/commit/2d2c83c26d4217446918f39d5cd5838e9ac27b32.patch |  git apply\n./boot.sh\n./configure --prefix=/usr/ --localstatedir=/var --enable-ssl --with-linux=/lib/modules/$(uname -r)/build\nmake -j `nproc`\nmake install\nmake modules_install\n\ncat &gt; /etc/depmod.d/openvswitch.conf &lt;&lt; EOF\noverride openvswitch * extra\noverride vport-* * extra\nEOF\n\ndepmod -a\ncp debian/openvswitch-switch.init /etc/init.d/openvswitch-switch\n/etc/init.d/openvswitch-switch force-reload-kmod\n</code></pre> <p>\u82e5\u4e4b\u524d\u5df2\u7ecf\u542f\u52a8\u8fc7 Kube-OVN\uff0c\u65e7\u7248\u672c OVS \u6a21\u5757\u5df2\u52a0\u8f7d\u81f3\u5185\u6838\uff0c\u5efa\u8bae\u91cd\u542f\u673a\u5668\u91cd\u65b0\u52a0\u8f7d\u65b0\u7248\u5185\u6838\u6a21\u5757\u3002</p>"},{"location":"advance/performance-tuning/#stt","title":"\u4f7f\u7528 STT \u7c7b\u578b\u96a7\u9053","text":"<p>\u5e38\u89c1\u7684\u96a7\u9053\u5c01\u88c5\u534f\u8bae\u4f8b\u5982 Geneve \u548c Vxlan \u4f7f\u7528 UDP \u534f\u8bae\u5bf9\u6570\u636e\u5305\u8fdb\u884c\u5c01\u88c5\uff0c\u5728\u5185\u6838\u4e2d\u6709\u826f\u597d\u7684\u652f\u6301\u3002\u4f46\u662f\u5f53\u4f7f\u7528 UDP \u5c01\u88c5 TCP \u6570\u636e\u5305\u65f6\uff0c \u73b0\u4ee3\u64cd\u4f5c\u7cfb\u7edf\u548c\u7f51\u5361\u9488\u5bf9 TCP \u534f\u8bae\u7684\u4f18\u5316\u548c offload \u529f\u80fd\u5c06\u65e0\u6cd5\u987a\u5229\u5de5\u4f5c\uff0c\u5bfc\u81f4 TCP \u7684\u541e\u5410\u91cf\u51fa\u73b0\u663e\u8457\u4e0b\u964d\u3002\u5728\u865a\u62df\u5316\u573a\u666f\u4e0b\u7531\u4e8e CPU \u7684\u9650\u5236\uff0c TCP \u5927\u5305\u7684\u541e\u5410\u91cf\u751a\u81f3\u53ef\u80fd\u53ea\u6709\u5bbf\u4e3b\u673a\u7f51\u7edc\u7684\u5341\u5206\u4e4b\u4e00\u3002</p> <p>STT \u63d0\u4f9b\u4e86\u4e00\u79cd\u521b\u65b0\u5f0f\u7684\u4f7f\u7528 TCP \u683c\u5f0f\u6570\u636e\u5305\u8fdb\u884c\u5c01\u88c5\u7684\u96a7\u9053\u534f\u8bae\uff0c\u8be5\u5c01\u88c5\u53ea\u662f\u6a21\u62df\u4e86 TCP \u534f\u8bae\u7684\u5934\u90e8\u683c\u5f0f\uff0c\u5e76\u6ca1\u6709\u771f\u6b63\u5efa\u7acb TCP \u8fde\u63a5\uff0c\u4f46\u662f\u53ef\u4ee5 \u5145\u5206\u5229\u7528\u73b0\u4ee3\u64cd\u4f5c\u7cfb\u7edf\u548c\u7f51\u5361\u7684 TCP \u4f18\u5316\u80fd\u529b\u3002\u5728\u6211\u4eec\u7684\u6d4b\u8bd5\u4e2d TCP \u5927\u5305\u7684\u541e\u5410\u91cf\u80fd\u6709\u6570\u500d\u7684\u63d0\u5347\uff0c\u8fbe\u5230\u63a5\u8fd1\u5bbf\u4e3b\u673a\u7f51\u7edc\u7684\u6027\u80fd\u6c34\u5e73\u3002</p> <p>STT \u96a7\u9053\u5e76\u6ca1\u6709\u9884\u5b89\u88c5\u5728\u5185\u6838\u5185\uff0c\u9700\u8981\u901a\u8fc7\u7f16\u8bd1 OVS \u5185\u6838\u6a21\u5757\u6765\u5b89\u88c5\uff0cOVS \u5185\u6838\u6a21\u5757\u7684\u7f16\u8bd1\u65b9\u6cd5\u53ef\u4ee5\u53c2\u8003\u4e0a\u4e00\u8282\u3002</p> <p>STT \u96a7\u9053\u5f00\u542f\uff1a</p> <pre><code>kubectl set env daemonset/ovs-ovn -n kube-system TUNNEL_TYPE=stt\n\nkubectl delete pod -n kube-system -lapp=ovs\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/security-group/","title":"SecurityGroup \u4f7f\u7528","text":"<p>Kube-OVN \u652f\u6301\u4e86\u5b89\u5168\u7ec4\u7684\u914d\u7f6e\uff0c\u914d\u7f6e\u5b89\u5168\u7ec4\u4f7f\u7528\u7684 CRD \u4e3a SecurityGroup\u3002</p>"},{"location":"advance/security-group/#_1","title":"\u5b89\u5168\u7ec4\u793a\u4f8b","text":"<pre><code>apiVersion: kubeovn.io/v1\nkind: SecurityGroup\nmetadata:\nname: sg-example\nspec:\nallowSameGroupTraffic: true\negressRules:\n- ipVersion: ipv4\npolicy: allow\npriority: 1\nprotocol: all\nremoteAddress: 10.16.0.13/16\nremoteType: address\ningressRules:\n- ipVersion: ipv4\npolicy: deny\npriority: 1\nprotocol: icmp\nremoteAddress: 10.16.0.14/16\nremoteType: address\n</code></pre> <p>\u5b89\u5168\u7ec4\u5404\u5b57\u6bb5\u7684\u5177\u4f53\u542b\u4e49\uff0c\u53ef\u4ee5\u53c2\u8003 Kube-OVN \u63a5\u53e3\u89c4\u8303\u3002</p> <p>Pod \u901a\u8fc7\u6dfb\u52a0 annotation \u6765\u7ed1\u5b9a\u5b89\u5168\u7ec4\uff0c\u4f7f\u7528\u7684 annotation \u6709\u4e24\u4e2a\uff1a</p> <pre><code>    ovn.kubernetes.io/port_security: \"true\"\novn.kubernetes.io/security_groups: sg-example\n</code></pre>"},{"location":"advance/security-group/#_2","title":"\u6ce8\u610f\u4e8b\u9879","text":"<ul> <li> <p>\u5b89\u5168\u7ec4\u6700\u540e\u662f\u901a\u8fc7\u8bbe\u7f6e ACL \u89c4\u5219\u6765\u9650\u5236\u8bbf\u95ee\u7684\uff0cOVN \u6587\u6863\u4e2d\u63d0\u5230\uff0c\u5982\u679c\u5339\u914d\u5230\u7684\u4e24\u4e2a ACL \u89c4\u5219\u62e5\u6709\u76f8\u540c\u7684\u4f18\u5148\u7ea7\uff0c\u5b9e\u9645\u8d77\u4f5c\u7528\u7684\u662f\u54ea\u4e2a ACL \u662f\u4e0d\u786e\u5b9a\u7684\u3002\u56e0\u6b64\u8bbe\u7f6e\u5b89\u5168\u7ec4\u89c4\u5219\u7684\u65f6\u5019\uff0c\u9700\u8981\u6ce8\u610f\u533a\u5206\u4f18\u5148\u7ea7\u3002</p> </li> <li> <p>\u5f53\u6dfb\u52a0\u5b89\u5168\u7ec4\u7684\u65f6\u5019\uff0c\u8981\u6e05\u695a\u7684\u77e5\u9053\u662f\u5728\u6dfb\u52a0\u4ec0\u4e48\u9650\u5236\u3002Kube-OVN \u4f5c\u4e3a CNI\uff0c\u521b\u5efa Pod \u540e\u4f1a\u8fdb\u884c Pod \u5230\u7f51\u5173\u7684\u8fde\u901a\u6027\u6d4b\u8bd5\uff0c\u5982\u679c\u8bbf\u95ee\u4e0d\u901a\u7f51\u5173\uff0c\u5c31\u4f1a\u5bfc\u81f4 Pod \u4e00\u76f4\u5904\u4e8e ContainerCreating \u72b6\u6001\uff0c\u65e0\u6cd5\u987a\u5229\u5207\u6362\u5230 Running \u72b6\u6001\u3002</p> </li> </ul>"},{"location":"advance/security-group/#_3","title":"\u5b9e\u9645\u6d4b\u8bd5","text":"<p>\u5229\u7528\u4ee5\u4e0b yaml \u521b\u5efa Pod\uff0c\u5728 annotation \u4e2d\u6307\u5b9a\u7ed1\u5b9a\u793a\u4f8b\u4e2d\u7684\u5b89\u5168\u7ec4\uff1a</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nlabels:\napp: static\nannotations:\novn.kubernetes.io/port_security: 'true'\novn.kubernetes.io/security_groups: 'sg-example'\nname: sg-test-pod\nnamespace: default\nspec:\nnodeName: kube-ovn-worker\ncontainers:\n- image: docker.io/library/nginx:alpine\nimagePullPolicy: IfNotPresent\nname: qatest\n</code></pre> <p>\u5b9e\u9645\u6d4b\u8bd5\u7ed3\u679c\u663e\u793a\u5982\u4e0b\uff1a</p> <pre><code># kubectl get pod -o wide\nNAME                   READY   STATUS              RESTARTS   AGE     IP           NODE                     NOMINATED NODE   READINESS GATES\nsg-test-pod            0/1     ContainerCreating   0          5h32m   &lt;none&gt;       kube-ovn-worker          &lt;none&gt;           &lt;none&gt;\ntest-99fff7f86-52h9r   1/1     Running             0          5h41m   10.16.0.14   kube-ovn-control-plane   &lt;none&gt;           &lt;none&gt;\ntest-99fff7f86-qcgjw   1/1     Running             0          5h43m   10.16.0.13   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;\n</code></pre> <p>\u6267\u884c kubectl describe pod \u67e5\u770b Pod \u7684\u4fe1\u606f\uff0c\u53ef\u4ee5\u770b\u5230\u62a5\u9519\u63d0\u793a\uff1a</p> <pre><code># kubectl describe pod sg-test-pod\nName:         sg-test-pod\nNamespace:    default\nPriority:     0\nNode:         kube-ovn-worker/172.18.0.2\nStart Time:   Tue, 28 Feb 2023 10:29:36 +0800\nLabels:       app=static\nAnnotations:  ovn.kubernetes.io/allocated: true\novn.kubernetes.io/cidr: 10.16.0.0/16\n              ovn.kubernetes.io/gateway: 10.16.0.1\n              ovn.kubernetes.io/ip_address: 10.16.0.15\n              ovn.kubernetes.io/logical_router: ovn-cluster\n              ovn.kubernetes.io/logical_switch: ovn-default\n              ovn.kubernetes.io/mac_address: 00:00:00:FA:17:97\n              ovn.kubernetes.io/pod_nic_type: veth-pair\n              ovn.kubernetes.io/port_security: true\novn.kubernetes.io/routed: true\novn.kubernetes.io/security_groups: sg-allow-reject\nStatus:       Pending\nIP:\nIPs:          &lt;none&gt;\n\u00b7\n\u00b7\n\u00b7\nEvents:\n  Type     Reason                  Age                    From     Message\n  ----     ------                  ----                   ----     -------\n  Warning  FailedCreatePodSandBox  5m3s (x70 over 4h59m)  kubelet  (combined from similar events): Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox \"40636e0c7f1ade5500fa958486163d74f2e2300051a71522a9afd7ba0538afb6\": plugin type=\"kube-ovn\" failed (add): RPC failed; request ip return 500 configure nic failed 10.16.0.15 network not ready after 200 ping 10.16.0.1\n</code></pre> <p>\u4fee\u6539\u5b89\u5168\u7ec4\u7684\u89c4\u5219\uff0c\u6dfb\u52a0\u5230\u7f51\u5173\u7684\u8bbf\u95ee\u89c4\u5219\uff0c\u53c2\u8003\u5982\u4e0b\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: SecurityGroup\nmetadata:\nname: sg-gw-both\nspec:\nallowSameGroupTraffic: true\negressRules:\n- ipVersion: ipv4\npolicy: allow\npriority: 2\nprotocol: all\nremoteAddress: 10.16.0.13/16\nremoteType: address\n- ipVersion: ipv4\npolicy: allow\npriority: 1\nprotocol: all\nremoteAddress: 10.16.0.1/16\nremoteType: address\ningressRules:\n- ipVersion: ipv4\npolicy: deny\npriority: 2\nprotocol: icmp\nremoteAddress: 10.16.0.14/16\nremoteType: address\n- ipVersion: ipv4\npolicy: allow\npriority: 1\nprotocol: icmp\nremoteAddress: 10.16.0.1/16\nremoteType: address\n</code></pre> <p>\u5206\u522b\u5728\u5165\u65b9\u5411\u548c\u51fa\u65b9\u5411\u89c4\u5219\u4e2d\uff0c\u6dfb\u52a0\u5141\u8bb8\u5230\u7f51\u5173\u7684\u8bbf\u95ee\u89c4\u5219\uff0c\u5e76\u4e14\u8bbe\u7f6e\u8be5\u89c4\u5219\u7684\u4f18\u5148\u7ea7\u6700\u9ad8\u3002</p> <p>\u5229\u7528\u4ee5\u4e0b yaml \u7ed1\u5b9a\u5b89\u5168\u7ec4\uff0c\u90e8\u7f72 Pod \u540e\uff0c\u786e\u8ba4 Pod \u53ef\u4ee5\u6b63\u5e38\u8fd0\u884c\uff1a</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nlabels:\napp: static\nannotations:\novn.kubernetes.io/port_security: 'true'\novn.kubernetes.io/security_groups: 'sg-gw-both'\nname: sg-gw-both\nnamespace: default\nspec:\nnodeName: kube-ovn-worker\ncontainers:\n- image: docker.io/library/nginx:alpine\nimagePullPolicy: IfNotPresent\nname: qatest\n</code></pre> <p>\u90e8\u7f72\u540e\u67e5\u770b Pod \u4fe1\u606f\uff1a</p> <pre><code># kubectl get pod -o wide\nNAME                   READY   STATUS              RESTARTS   AGE     IP           NODE                     NOMINATED NODE   READINESS GATES\nsg-test-pod            0/1     ContainerCreating   0          5h41m   &lt;none&gt;       kube-ovn-worker          &lt;none&gt;           &lt;none&gt;\nsg-gw-both             1/1     Running             0          5h37m   10.16.0.19   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;\n</code></pre> <p>\u56e0\u6b64\u5bf9\u4e8e\u5b89\u5168\u7ec4\u7684\u4f7f\u7528\uff0c\u8981\u7279\u522b\u660e\u786e\u6dfb\u52a0\u7684\u9650\u5236\u89c4\u5219\u7684\u4f5c\u7528\u3002\u5982\u679c\u5355\u7eaf\u662f\u9650\u5236\u6d41\u91cf\u8bbf\u95ee\uff0c\u53ef\u4ee5\u8003\u8651\u4f7f\u7528\u7f51\u7edc\u7b56\u7565\u5b9e\u73b0\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/vip/","title":"VIP \u9884\u7559\u8bbe\u7f6e","text":"<p>\u5728\u4e00\u4e9b\u573a\u666f\u4e0b\u6211\u4eec\u5e0c\u671b\u52a8\u6001\u7684\u9884\u7559\u4e00\u90e8\u5206 IP \u4f46\u662f\u5e76\u4e0d\u5206\u914d\u7ed9 Pod \u800c\u662f\u5206\u914d\u7ed9\u5176\u4ed6\u7684\u57fa\u7840\u8bbe\u65bd\u542f\u7528\uff0c\u4f8b\u5982\uff1a</p> <ul> <li>Kubernetes \u5d4c\u5957 Kubernetes \u7684\u573a\u666f\u4e2d\u4e0a\u5c42 Kubernetes \u4f7f\u7528 Underlay \u7f51\u7edc\u4f1a\u5360\u7528\u5e95\u5c42 Subnet \u53ef\u7528\u5730\u5740\u3002</li> <li>LB \u6216\u5176\u4ed6\u7f51\u7edc\u57fa\u7840\u8bbe\u65bd\u9700\u8981\u4f7f\u7528\u4e00\u4e2a Subnet \u5185\u7684 IP\uff0c\u4f46\u4e0d\u4f1a\u5355\u72ec\u8d77 Pod\u3002</li> </ul>"},{"location":"advance/vip/#vip_1","title":"\u521b\u5efa\u968f\u673a\u5730\u5740 VIP","text":"<p>\u5982\u679c\u53ea\u662f\u4e3a\u4e86\u9884\u7559\u82e5\u5e72 IP \u800c\u5bf9 IP \u5730\u5740\u672c\u8eab\u6ca1\u6709\u8981\u6c42\u53ef\u4ee5\u4f7f\u7528\u4e0b\u9762\u7684 yaml \u8fdb\u884c\u521b\u5efa\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Vip\nmetadata:\nname: vip-dynamic-01\nspec:\nsubnet: ovn-default\ntype: \"\"\n</code></pre> <ul> <li><code>subnet</code>: \u5c06\u4ece\u8be5 Subnet \u4e2d\u9884\u7559 IP\u3002</li> <li><code>type</code>: \u76ee\u524d\u652f\u6301\u4e24\u79cd\u7c7b\u578b\uff0c\u4e3a\u7a7a\u8868\u793a\u4ec5\u7528\u4e8e ipam ip \u5360\u4f4d\uff0c<code>switch_lb_vip</code> \u8868\u793a\u8be5 vip \u4ec5\u7528\u4e8e switch lb \u524d\u7aef vip \u548c\u540e\u7aef ip \u9700\u5904\u4e8e\u540c\u4e00\u5b50\u7f51\u3002</li> </ul> <p>\u521b\u5efa\u6210\u529f\u540e\u67e5\u8be2\u8be5 VIP\uff1a</p> <pre><code># kubectl get vip\nNAME             V4IP         PV4IP   MAC                 PMAC   V6IP   PV6IP   SUBNET        READY\nvip-dynamic-01   10.16.0.12           00:00:00:F0:DB:25                         ovn-default   true\n</code></pre> <p>\u53ef\u89c1\u8be5 VIP \u88ab\u5206\u914d\u4e86 <code>10.16.0.12</code> \u7684 IP \u5730\u5740\uff0c\u8be5\u5730\u5740\u53ef\u4ee5\u4e4b\u540e\u4f9b\u5176\u4ed6\u7f51\u7edc\u57fa\u7840\u8bbe\u65bd\u4f7f\u7528\u3002</p>"},{"location":"advance/vip/#vip_2","title":"\u521b\u5efa\u56fa\u5b9a\u5730\u5740 VIP","text":"<p>\u5982\u5bf9\u9884\u7559\u7684 VIP \u7684 IP \u5730\u5740\u6709\u9700\u6c42\u53ef\u4f7f\u7528\u4e0b\u9762\u7684 yaml \u8fdb\u884c\u56fa\u5b9a\u5206\u914d\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Vip\nmetadata:\nname: static-vip01\nspec:\nsubnet: ovn-default v4ip: \"10.16.0.121\"\n</code></pre> <ul> <li><code>subnet</code>: \u5c06\u4ece\u8be5 Subnet \u4e2d\u9884\u7559 IP\u3002</li> <li><code>v4ip</code>: \u56fa\u5b9a\u5206\u914d\u7684 IP \u5730\u5740\uff0c\u8be5\u5730\u5740\u9700\u5728 <code>subnet</code> \u7684 CIDR \u8303\u56f4\u5185\u3002</li> </ul> <p>\u521b\u5efa\u6210\u529f\u540e\u67e5\u8be2\u8be5 VIP\uff1a</p> <pre><code># kubectl get vip\nNAME             V4IP         PV4IP   MAC                 PMAC   V6IP   PV6IP   SUBNET        READY\nstatic-vip01   10.16.0.121           00:00:00:F0:DB:26                         ovn-default   true\n</code></pre> <p>\u53ef\u89c1\u8be5 VIP \u88ab\u5206\u914d\u4e86\u6240\u9884\u671f\u7684 IP \u5730\u5740\u3002</p>"},{"location":"advance/vip/#pod-vip-ip","title":"Pod \u4f7f\u7528 VIP \u6765\u56fa\u5b9a IP","text":"<p>\u8be5\u529f\u80fd\u4ece v1.12 \u5f00\u59cb\u652f\u6301\u3002</p> <p>\u53ef\u4ee5\u4f7f\u7528 annotation \u5c06\u67d0\u4e2a VIP \u5206\u914d\u7ed9\u4e00\u4e2a Pod\uff1a</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: static-ip\nannotations:\novn.kubernetes.io/vip: vip-dynamic-01 # \u6307\u5b9a vip\nnamespace: default\nspec:\ncontainers:\n- name: static-ip\nimage: docker.io/library/nginx:alpine\n</code></pre>"},{"location":"advance/vip/#statefulset-kubevirt-vm-vip","title":"StatefulSet \u548c Kubevirt VM \u4fdd\u7559 VIP","text":"<p>\u9488\u5bf9 <code>StatefulSet</code> \u548c <code>VM</code> \u7684\u7279\u6b8a\u6027\uff0c\u5728\u4ed6\u4eec\u7684 Pod \u9500\u6bc1\u518d\u62c9\u8d77\u8d77\u540e\u4f1a\u91cd\u65b0\u4f7f\u7528\u4e4b\u524d\u8bbe\u7f6e\u7684 VIP\u3002</p> <p>VM \u4fdd\u7559 VIP \u9700\u8981\u786e\u4fdd <code>kube-ovn-controller</code> \u7684 <code>keep-vm-ip</code> \u53c2\u6570\u4e3a <code>true</code>\u3002\u8bf7\u53c2\u8003 Kubevirt VM \u56fa\u5b9a\u5730\u5740\u5f00\u542f\u8bbe\u7f6e</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/vpc-dns/","title":"\u81ea\u5b9a\u4e49 VPC DNS","text":"<p>\u7531\u4e8e\u7528\u6237\u81ea\u5b9a\u4e49 VPC \u548c \u9ed8\u8ba4 VPC \u7f51\u7edc\u76f8\u4e92\u9694\u79bb\uff0c\u81ea\u5b9a VPC \u5185\u65e0\u6cd5\u8bbf\u95ee\u5230\u90e8\u7f72\u5728\u9ed8\u8ba4 VPC \u5185\u7684 coredns\u3002 \u5982\u679c\u7528\u6237\u5e0c\u671b\u5728\u81ea\u5b9a\u4e49 VPC \u5185\u4f7f\u7528 Kubernetes \u63d0\u4f9b\u7684\u96c6\u7fa4\u5185\u57df\u540d\u89e3\u6790\u80fd\u529b\uff0c\u53ef\u4ee5\u53c2\u8003\u672c\u6587\u6863\uff0c\u5229\u7528 <code>vpc-dns</code> CRD \u6765\u5b9e\u73b0\u3002</p> <p>\u8be5 CRD \u6700\u7ec8\u4f1a\u90e8\u7f72\u4e00\u4e2a coredns\uff0c\u8be5 Pod \u6709\u4e24\u4e2a\u7f51\u5361\uff0c\u4e00\u4e2a\u7f51\u5361\u5728\u7528\u6237\u81ea\u5b9a\u4e49 VPC\uff0c\u53e6\u4e00\u4e2a\u7f51\u5361\u5728\u9ed8\u8ba4 VPC \u4ece\u800c\u5b9e\u73b0\u7f51\u7edc\u4e92\u901a\uff0c\u540c\u65f6\u901a\u8fc7\u81ea\u5b9a\u4e49 VPC \u5185\u90e8\u8d1f\u8f7d\u5747\u8861\u63d0\u4f9b\u81ea\u5b9a\u4e49 VPC \u5185\u7684\u4e00\u4e2a\u5185\u90e8\u8d1f\u8f7d\u5747\u8861\u3002</p>"},{"location":"advance/vpc-dns/#vpc-dns_1","title":"\u90e8\u7f72 vpc-dns \u6240\u4f9d\u8d56\u7684\u8d44\u6e90","text":"<pre><code>apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\nlabels:\nkubernetes.io/bootstrapping: rbac-defaults\nname: system:vpc-dns\nrules:\n- apiGroups:\n- \"\"\nresources:\n- endpoints\n- services\n- pods\n- namespaces\nverbs:\n- list\n- watch\n- apiGroups:\n- discovery.k8s.io\nresources:\n- endpointslices\nverbs:\n- list\n- watch\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\nannotations:\nrbac.authorization.kubernetes.io/autoupdate: \"true\"\nlabels:\nkubernetes.io/bootstrapping: rbac-defaults\nname: vpc-dns\nroleRef:\napiGroup: rbac.authorization.k8s.io\nkind: ClusterRole\nname: system:vpc-dns\nsubjects:\n- kind: ServiceAccount\nname: vpc-dns\nnamespace: kube-system\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\nname: vpc-dns\nnamespace: kube-system\n---\napiVersion: v1\nkind: ConfigMap\nmetadata:\nname: vpc-dns-corefile\nnamespace: kube-system\ndata:\nCorefile: |\n.:53 {\nerrors\nhealth {\nlameduck 5s\n}\nready\nkubernetes cluster.local in-addr.arpa ip6.arpa {\npods insecure\nfallthrough in-addr.arpa ip6.arpa\n}\nprometheus :9153\nforward . /etc/resolv.conf {\nprefer_udp\n}\ncache 30\nloop\nreload\nloadbalance\n}\n</code></pre> <p>\u9664\u4e86\u4ee5\u4e0a\u8d44\u6e90\uff0c\u8be5\u529f\u80fd\u8fd8\u4f9d\u8d56 nat-gw-pod \u955c\u50cf\u8fdb\u884c\u8def\u7531\u914d\u7f6e\u3002</p>"},{"location":"advance/vpc-dns/#_1","title":"\u914d\u7f6e\u9644\u52a0\u7f51\u5361","text":"<pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: ovn-nad\nnamespace: default\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.0\",\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"ovn-nad.default.ovn\"\n}'\n</code></pre>"},{"location":"advance/vpc-dns/#ovn-default-provider","title":"\u4fee\u6539 ovn-default \u5b50\u7f51\u7684 provider","text":"<p>\u4fee\u6539 ovn-default \u7684 provider\uff0c\u4e3a\u4e0a\u9762 nad \u914d\u7f6e\u7684 provider <code>ovn-nad.default.ovn</code></p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: ovn-default\nspec:\ncidrBlock: 10.16.0.0/16\ndefault: true\ndisableGatewayCheck: false\ndisableInterConnection: false\nenableDHCP: false\nenableIPv6RA: false\nexcludeIps:\n- 10.16.0.1\ngateway: 10.16.0.1\ngatewayType: distributed\nlogicalGateway: false\nnatOutgoing: true\nprivate: false\nprotocol: IPv4\nprovider: ovn-nad.default.ovn # \u53ea\u9700\u4fee\u6539\u8be5\u5b57\u6bb5\nvpc: ovn-cluster\n</code></pre>"},{"location":"advance/vpc-dns/#vpc-dns-configmap","title":"\u914d\u7f6e vpc-dns \u7684 Configmap","text":"<p>\u5728 kube-system \u547d\u540d\u7a7a\u95f4\u4e0b\u521b\u5efa configmap\uff0c\u914d\u7f6e vpc-dns \u4f7f\u7528\u53c2\u6570\uff0c\u7528\u4e8e\u540e\u9762\u542f\u52a8 vpc-dns \u529f\u80fd\uff1a</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: vpc-dns-config\nnamespace: kube-system\ndata:\ncoredns-vip: 10.96.0.3\nenable-vpc-dns: \"true\"\nnad-name: ovn-nad\nnad-provider: ovn-nad.default.ovn\n</code></pre> <ul> <li><code>enable-vpc-dns</code>\uff1a\u662f\u5426\u542f\u7528\u529f\u80fd\uff0c\u9ed8\u8ba4 <code>true</code>\u3002</li> <li><code>coredns-image</code>\uff1adns \u90e8\u7f72\u955c\u50cf\u3002\u9ed8\u8ba4\u4e3a\u96c6\u7fa4 coredns \u90e8\u7f72\u7248\u672c\u3002</li> <li><code>coredns-vip</code>\uff1a\u4e3a coredns \u63d0\u4f9b lb \u670d\u52a1\u7684 vip\u3002</li> <li><code>coredns-template</code>\uff1acoredns \u90e8\u7f72\u6a21\u677f\u6240\u5728\u7684 URL\u3002\u9ed8\u8ba4\u83b7\u53d6\u5f53\u524d\u7248\u672c ovn \u76ee\u5f55\u4e0b <code>coredns-template.yaml</code> \u9ed8\u8ba4\u4e3a <code>https://raw.githubusercontent.com/kubeovn/kube-ovn/\u5f53\u524d\u7248\u672c/yamls/coredns-template.yaml</code> \u3002</li> <li><code>nad-name</code>\uff1a\u914d\u7f6e\u7684 <code>network-attachment-definitions</code> \u8d44\u6e90\u540d\u79f0\u3002</li> <li><code>nad-provider</code>\uff1a\u4f7f\u7528\u7684 provider \u540d\u79f0\u3002</li> <li><code>k8s-service-host</code>\uff1a\u7528\u4e8e coredns \u8bbf\u95ee k8s apiserver \u670d\u52a1\u7684 ip\uff0c\u9ed8\u8ba4\u4e3a\u96c6\u7fa4\u5185 apiserver \u5730\u5740\u3002</li> <li><code>k8s-service-port</code>\uff1a\u7528\u4e8e coredns \u8bbf\u95ee k8s apiserver \u670d\u52a1\u7684 port\uff0c\u9ed8\u8ba4\u4e3a\u96c6\u7fa4\u5185 apiserver \u7aef\u53e3\u3002</li> </ul>"},{"location":"advance/vpc-dns/#vpc-dns_2","title":"\u90e8\u7f72 vpc-dns","text":"<p>\u914d\u7f6e vpc-dns yaml\uff1a</p> <pre><code>kind: VpcDns\napiVersion: kubeovn.io/v1\nmetadata:\nname: test-cjh1\nspec:\nvpc: cjh-vpc-1\nsubnet: cjh-subnet-1\n</code></pre> <ul> <li><code>vpc</code> \uff1a \u7528\u4e8e\u90e8\u7f72 dns \u7ec4\u4ef6\u7684 vpc \u540d\u79f0\u3002</li> <li><code>subnet</code>\uff1a\u7528\u4e8e\u90e8\u7f72 dns \u7ec4\u4ef6\u7684\u5b50\u540d\u79f0\u3002</li> </ul> <p>\u67e5\u770b\u90e8\u7f72\u8d44\u6e90\u7684\u4fe1\u606f\uff1a</p> <pre><code># kubectl get vpc-dns\nNAME        ACTIVE   VPC         SUBNET   \ntest-cjh1   false    cjh-vpc-1   cjh-subnet-1   \ntest-cjh2   true     cjh-vpc-1   cjh-subnet-2 </code></pre> <p><code>ACTIVE</code> : <code>true</code> \u90e8\u7f72\u4e86\u81ea\u5b9a\u4e49 dns \u7ec4\u4ef6\uff0c<code>false</code> \u65e0\u90e8\u7f72\u3002</p> <ul> <li>\u9650\u5236\uff1a\u4e00\u4e2a VPC \u4e0b\u53ea\u4f1a\u90e8\u7f72\u4e00\u4e2a\u81ea\u5b9a\u4e49 dns \u7ec4\u4ef6;</li> <li>\u5f53\u4e00\u4e2a VPC \u4e0b\u914d\u7f6e\u591a\u4e2a vpc-dns \u8d44\u6e90\uff08\u5373\u540c\u4e00\u4e2a VPC \u4e0d\u540c\u7684 subnet\uff09\uff0c\u53ea\u6709\u4e00\u4e2a vpc-dns \u8d44\u6e90\u72b6\u6001 <code>true</code>\uff0c\u5176\u4ed6\u4e3a <code>fasle</code>;</li> <li>\u5f53 <code>true</code> \u7684 vpc-dns \u88ab\u5220\u9664\u6389\uff0c\u4f1a\u83b7\u53d6\u5176\u4ed6 <code>false</code> \u7684 vpc-dns \u8fdb\u884c\u90e8\u7f72\u3002</li> </ul>"},{"location":"advance/vpc-dns/#_2","title":"\u9a8c\u8bc1\u90e8\u7f72\u7ed3\u679c","text":"<p>\u67e5\u770b vpc-dns Pod \u72b6\u6001\uff0c\u4f7f\u7528 label <code>app=vpc-dns</code>\uff0c\u53ef\u4ee5\u67e5\u770b\u6240\u6709 vpc-dns pod \u72b6\u6001\uff1a</p> <pre><code># kubectl -n kube-system get pods -l app=vpc-dns\nNAME                                 READY   STATUS    RESTARTS   AGE\nvpc-dns-test-cjh1-7b878d96b4-g5979   1/1     Running   0          28s\nvpc-dns-test-cjh1-7b878d96b4-ltmf9   1/1     Running   0          28s\n</code></pre> <p>\u67e5\u770b slr \u72b6\u6001\u4fe1\u606f\uff1a</p> <pre><code># kubectl -n kube-system get slr\nNAME                VIP         PORT(S)                  SERVICE                             AGE\nvpc-dns-test-cjh1   10.96.0.3   53/UDP,53/TCP,9153/TCP   kube-system/slr-vpc-dns-test-cjh1   113s\n</code></pre> <p>\u8fdb\u5165\u8be5 VPC \u4e0b\u7684 Pod\uff0c\u6d4b\u8bd5 dns \u89e3\u6790:</p> <pre><code>nslookup kubernetes.default.svc.cluster.local 10.96.0.3\n</code></pre> <p>\u8be5 VPC \u4e0b\u7684 switch lb rule \u6240\u5728\u7684\u5b50\u7f51\u4ee5\u53ca\u540c\u4e00 VPC \u4e0b\u7684\u5176\u4ed6\u5b50\u7f51\u4e0b\u7684 pod \u90fd\u53ef\u4ee5\u89e3\u6790\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/vpc-internal-lb/","title":"\u81ea\u5b9a\u4e49 VPC \u5185\u90e8\u8d1f\u8f7d\u5747\u8861","text":"<p>Kubernetes \u63d0\u4f9b\u7684 Service \u53ef\u4ee5\u7528\u4f5c\u96c6\u7fa4\u5185\u7684\u8d1f\u8f7d\u5747\u8861\uff0c \u4f46\u662f\u5728\u81ea\u5b9a\u4e49 VPC \u6a21\u5f0f\u4e0b\uff0c \u4f7f\u7528 Service \u4f5c\u4e3a\u5185\u90e8\u8d1f\u8f7d\u5747\u8861\u5b58\u5728\u5982\u4e0b\u51e0\u4e2a\u95ee\u9898\uff1a</p> <ol> <li>Service IP \u8303\u56f4\u4e3a\u96c6\u7fa4\u8d44\u6e90\uff0c\u6240\u6709\u81ea\u5b9a\u4e49 VPC \u5171\u4eab\uff0c\u65e0\u6cd5\u91cd\u53e0\u3002</li> <li>\u7528\u6237\u65e0\u6cd5\u6309\u7167\u81ea\u5df1\u610f\u613f\u8bbe\u7f6e\u5185\u90e8\u8d1f\u8f7d\u5747\u8861\u7684 IP \u5730\u5740\u3002</li> </ol> <p>\u4e3a\u4e86\u89e3\u51b3\u4e0a\u8ff0\u95ee\u9898\uff0cKube-OVN \u5728 1.11 \u5f15\u5165 <code>SwitchLBRule</code> CRD\uff0c\u7528\u6237\u53ef\u4ee5\u8bbe\u7f6e\u81ea\u5b9a\u4e49 VPC \u5185\u7684\u5185\u90e8\u8d1f\u8f7d\u5747\u8861\u89c4\u5219\u3002</p> <p><code>SwitchLBRule</code> \u652f\u6301\u4ee5\u4e0b\u4e24\u79cd\u65b9\u5f0f\u8bbe\u7f6e\u81ea\u5b9a\u4e49 VPC \u5185\u7684\u5185\u90e8\u8d1f\u8f7d\u5747\u8861\u89c4\u5219\u3002</p>"},{"location":"advance/vpc-internal-lb/#selector","title":"<code>Selector</code> \u81ea\u52a8\u751f\u6210\u8d1f\u8f7d\u5747\u8861\u89c4\u5219","text":"<p>\u901a\u8fc7 <code>selector</code> \u53ef\u4ee5\u901a\u8fc7 <code>label</code> \u81ea\u52a8\u5173\u8054 <code>pod</code> \u914d\u7f6e\u751f\u6210\u8d1f\u8f7d\u5747\u8861\u89c4\u5219\u3002</p> <p><code>SwitchLBRule</code> \u6837\u4f8b\u5982\u4e0b\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: SwitchLBRule\nmetadata:\nname:  cjh-slr-nginx\nspec:\nvip: 1.1.1.1\nsessionAffinity: ClientIP\nnamespace: default\nselector:\n- app:nginx\nports:\n- name: dns\nport: 8888\ntargetPort: 80\nprotocol: TCP\n</code></pre> <ul> <li> <p><code>selector</code>, <code>sessionAffinity</code> \u548c <code>port</code> \u4f7f\u7528\u65b9\u5f0f\u540c Kubernetes Service\u3002</p> </li> <li> <p><code>vip</code>\uff1a\u81ea\u5b9a\u4e49\u8d1f\u8f7d\u5747\u8861\u7684 IP \u5730\u5740\u3002</p> </li> <li> <p><code>namespace</code>\uff1a<code>selector</code> \u6240\u9009\u62e9 Pod \u6240\u5728\u547d\u540d\u7a7a\u95f4\u3002</p> <p>Kube-OVN \u4f1a\u6839\u636e <code>SwitchLBRule</code> \u5b9a\u4e49\u9009\u62e9\u7684 Pod \u5f97\u51fa Pod \u6240\u5728 VPC \u5e76\u8bbe\u7f6e\u5bf9\u5e94\u7684 L2 LB\u3002</p> </li> </ul>"},{"location":"advance/vpc-internal-lb/#endpoints","title":"<code>Endpoints</code> \u81ea\u5b9a\u4e49\u8d1f\u8f7d\u5747\u8861\u89c4\u5219","text":"<p>\u901a\u8fc7 <code>endpoints</code> \u53ef\u4ee5\u81ea\u5b9a\u4e49\u8d1f\u8f7d\u5747\u8861\u89c4\u5219\uff0c\u7528\u4ee5\u652f\u6301\u65e0\u6cd5\u901a\u8fc7 <code>selector</code> \u81ea\u52a8\u751f\u6210\u8d1f\u8f7d\u5747\u8861\u89c4\u5219\u7684\u573a\u666f\uff0c\u6bd4\u5982\u8d1f\u8f7d\u5747\u8861\u540e\u7aef\u662f <code>kubevirt</code> \u521b\u5efa\u7684 <code>vm</code> \u3002</p> <p><code>SwitchLBRule</code> \u6837\u4f8b\u5982\u4e0b\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: SwitchLBRule\nmetadata:\nname:  cjh-slr-nginx\nspec:\nvip: 1.1.1.1\nsessionAffinity: ClientIP\nnamespace: default\nendpoints:\n- 192.168.0.101\n- 192.168.0.102\n- 192.168.0.103\nports:\n- name: dns\nport: 8888\ntargetPort: 80\nprotocol: TCP\n</code></pre> <ul> <li> <p><code>sessionAffinity</code> \u548c <code>port</code> \u4f7f\u7528\u65b9\u5f0f\u540c Kubernetes Service\u3002</p> </li> <li> <p><code>vip</code>\uff1a\u81ea\u5b9a\u4e49\u8d1f\u8f7d\u5747\u8861\u7684 IP \u5730\u5740\u3002</p> </li> <li> <p><code>namespace</code>\uff1a<code>selector</code> \u6240\u9009\u62e9 Pod \u6240\u5728\u547d\u540d\u7a7a\u95f4\u3002</p> </li> <li> <p><code>endpoints</code>\uff1a\u8d1f\u8f7d\u5747\u8861\u540e\u7aef IP \u5217\u8868\u3002</p> </li> </ul> <p>\u6ce8\uff1a\u5982\u679c\u540c\u65f6\u914d\u7f6e\u4e86 <code>selector</code> \u548c <code>endpoints</code>,\u4f1a\u81ea\u52a8\u5ffd\u7565<code>selector</code>\u914d\u7f6e\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/vpc-peering/","title":"VPC \u4e92\u8054","text":"<p>VPC \u4e92\u8054\u63d0\u4f9b\u4e86\u4e00\u79cd\u5c06\u4e24\u4e2a VPC \u7f51\u7edc\u901a\u8fc7\u903b\u8f91\u8def\u7531\u6253\u901a\u7684\u673a\u5236\uff0c\u4ece\u800c\u4f7f\u4e24\u4e2a VPC \u5185\u7684\u5de5\u4f5c\u8d1f\u8f7d\u53ef\u4ee5\u50cf\u5728\u540c\u4e00\u4e2a\u79c1\u6709\u7f51\u7edc\u4e00\u6837\uff0c \u901a\u8fc7\u79c1\u6709\u5730\u5740\u76f8\u4e92\u8bbf\u95ee\uff0c\u65e0\u9700\u901a\u8fc7\u5916\u90e8\u7f51\u5173\u8fdb\u884c NAT \u8f6c\u53d1\u3002</p>"},{"location":"advance/vpc-peering/#_1","title":"\u524d\u63d0\u6761\u4ef6","text":"<ol> <li>\u8be5\u529f\u80fd\u53ea\u9002\u7528\u4e8e\u7528\u6237\u81ea\u5b9a\u4e49 VPC\u3002</li> <li>\u4e3a\u4e86\u907f\u514d\u8def\u7531\u91cd\u53e0\u4e24\u4e2a VPC \u5185\u7684\u5b50\u7f51 CIDR \u4e0d\u80fd\u91cd\u53e0\u3002</li> <li>\u76ee\u524d\u53ea\u652f\u6301\u4e24\u4e2a VPC \u7684\u4e92\u8054\uff0c\u66f4\u591a\u7ec4 VPC \u4e4b\u95f4\u7684\u4e92\u8054\u6682\u4e0d\u652f\u6301\u3002</li> </ol>"},{"location":"advance/vpc-peering/#_2","title":"\u4f7f\u7528\u65b9\u5f0f","text":"<p>\u9996\u5148\u521b\u5efa\u4e24\u4e2a\u4e0d\u4e92\u8054\u7684 VPC\uff0c\u6bcf\u4e2a VPC \u4e0b\u5404\u6709\u4e00\u4e2a Subnet\uff0cSubnet \u7684 CIDR \u4e92\u4e0d\u91cd\u53e0\u3002</p> <pre><code>kind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: vpc-1\nspec: {}\n---\nkind: Subnet\napiVersion: kubeovn.io/v1\nmetadata:\nname: net1\nspec:\nvpc: vpc-1\ncidrBlock: 10.0.0.0/16\n---\nkind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: vpc-2\nspec: {}\n---\nkind: Subnet\napiVersion: kubeovn.io/v1\nmetadata:\nname: net2\nspec:\nvpc: vpc-2\ncidrBlock: 172.31.0.0/16\n</code></pre> <p>\u5728\u6bcf\u4e2a VPC \u5185\u5206\u522b\u589e\u52a0 <code>vpcPeerings</code> \u548c\u5bf9\u5e94\u7684\u9759\u6001\u8def\u7531\uff1a</p> <pre><code>kind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: vpc-1\nspec: vpcPeerings:\n- remoteVpc: vpc-2\nlocalConnectIP: 169.254.0.1/30\nstaticRoutes:\n- cidr: 172.31.0.0/16\nnextHopIP: 169.254.0.2\npolicy: policyDst\n---\nkind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: vpc-2\nspec:\nvpcPeerings:\n- remoteVpc: vpc-1\nlocalConnectIP: 169.254.0.2/30\nstaticRoutes:\n- cidr: 10.0.0.0/16\nnextHopIP: 169.254.0.1\npolicy: policyDst\n</code></pre> <ul> <li><code>remoteVpc</code>: \u4e92\u8054\u7684\u53e6\u4e00\u4e2a VPC \u7684\u540d\u5b57\u3002</li> <li><code>localConnectIP</code>: \u4f5c\u4e3a\u4e92\u8054\u7aef\u70b9\u7684 IP \u5730\u5740\u548c CIDR\uff0c\u6ce8\u610f\u4e24\u7aef IP \u5e94\u5c5e\u4e8e\u540c\u4e00 CIDR\uff0c\u4e14\u4e0d\u80fd\u548c\u5df2\u6709\u5b50\u7f51\u51b2\u7a81\u3002</li> <li><code>cidr</code>\uff1a\u53e6\u4e00\u7aef Subnet \u7684 CIDR\u3002</li> <li><code>nextHopIP</code>\uff1a\u4e92\u8054 VPC \u53e6\u4e00\u7aef\u7684 <code>localConnectIP</code>\u3002</li> </ul> <p>\u5206\u522b\u5728\u4e24\u4e2a Subnet \u4e0b\u521b\u5efa Pod</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nannotations:\novn.kubernetes.io/logical_switch: net1\nname: vpc-1-pod\nspec:\ncontainers:\n- name: vpc-1-pod\nimage: docker.io/library/nginx:alpine\n---\napiVersion: v1\nkind: Pod\nmetadata:\nannotations:\novn.kubernetes.io/logical_switch: net2\nname: vpc-2-pod\nspec:\ncontainers:\n- name: vpc-2-pod\nimage: docker.io/library/nginx:alpine\n</code></pre> <p>\u6d4b\u8bd5\u7f51\u7edc\u8fde\u901a\u6027</p> <pre><code># kubectl exec -it vpc-1-pod -- ping $(kubectl get pod vpc-2-pod -o jsonpath='{.status.podIP}')\nPING 172.31.0.2 (172.31.0.2): 56 data bytes\n64 bytes from 172.31.0.2: seq=0 ttl=62 time=0.655 ms\n64 bytes from 172.31.0.2: seq=1 ttl=62 time=0.086 ms\n64 bytes from 172.31.0.2: seq=2 ttl=62 time=0.098 ms\n^C\n--- 172.31.0.2 ping statistics ---\n3 packets transmitted, 3 packets received, 0% packet loss\nround-trip min/avg/max = 0.086/0.279/0.655 ms\n# kubectl exec -it vpc-2-pod -- ping $(kubectl get pod vpc-1-pod -o jsonpath='{.status.podIP}')\nPING 10.0.0.2 (10.0.0.2): 56 data bytes\n64 bytes from 10.0.0.2: seq=0 ttl=62 time=0.594 ms\n64 bytes from 10.0.0.2: seq=1 ttl=62 time=0.093 ms\n64 bytes from 10.0.0.2: seq=2 ttl=62 time=0.088 ms\n^C\n--- 10.0.0.2 ping statistics ---\n3 packets transmitted, 3 packets received, 0% packet loss\nround-trip min/avg/max = 0.088/0.258/0.594 ms\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/windows/","title":"Windows \u652f\u6301","text":"<p>Kube-OVN \u652f\u6301\u5305\u542b Windows \u7cfb\u7edf\u8282\u70b9\u7684 Kubernetes \u96c6\u7fa4\u7f51\u7edc\uff0c\u53ef\u4ee5\u5c06 Windows \u5bb9\u5668\u7684\u7f51\u7edc\u7edf\u4e00\u63a5\u5165\u8fdb\u884c\u7ba1\u7406\u3002</p>"},{"location":"advance/windows/#_1","title":"\u524d\u63d0\u6761\u4ef6","text":"<ul> <li>\u53c2\u8003 Adding Windows nodes \u589e\u52a0 Windows \u8282\u70b9\u3002</li> <li>Windows \u8282\u70b9\u5fc5\u987b\u5b89\u88c5 KB4489899 \u8865\u4e01\u4ee5\u4f7f Overlay/VXLAN \u7f51\u7edc\u6b63\u5e38\u5de5\u4f5c\uff0c\u5efa\u8bae\u66f4\u65b0\u7cfb\u7edf\u81f3\u6700\u65b0\u7248\u672c\u3002</li> <li>Windows \u8282\u70b9\u5fc5\u987b\u5b89\u88c5 Hyper-V \u53ca\u7ba1\u7406\u5de5\u5177\u3002</li> <li>\u7531\u4e8e Windows \u9650\u5236\u96a7\u9053\u5c01\u88c5\u53ea\u80fd\u4f7f\u7528 Vxlan \u6a21\u5f0f\u3002</li> <li>\u6682\u4e0d\u652f\u6301 SSL\uff0cIPv6\uff0c\u53cc\u6808\uff0cQoS \u529f\u80fd\u3002</li> <li>\u6682\u4e0d\u652f\u6301\u52a8\u6001\u5b50\u7f51\uff0c\u52a8\u6001\u96a7\u9053\u63a5\u53e3\u529f\u80fd\uff0c\u9700\u5728\u5b89\u88c5 Windows \u8282\u70b9\u524d\u5b8c\u6210\u5b50\u7f51\u521b\u5efa\uff0c\u5e76\u56fa\u5b9a\u7f51\u7edc\u63a5\u53e3\u3002</li> <li>\u4e0d\u652f\u6301\u591a\u4e2a <code>ProviderNetwork</code>\uff0c\u4e14\u65e0\u6cd5\u52a8\u6001\u8c03\u6574\u6865\u63a5\u63a5\u53e3\u914d\u7f6e\u3002</li> </ul>"},{"location":"advance/windows/#ovs","title":"\u5b89\u88c5 OVS","text":"<p>\u7531\u4e8e\u4e0a\u6e38 OVN \u548c OVS \u5bf9 Windows \u5bb9\u5668\u652f\u6301\u5b58\u5728\u4e00\u4e9b\u95ee\u9898\uff0c\u9700\u8981\u4f7f\u7528 Kube-OVN \u63d0\u4f9b\u7684\u7ecf\u8fc7\u4fee\u6539\u7684\u5b89\u88c5\u5305\u8fdb\u884c\u5b89\u88c5\u3002</p> <p>\u4f7f\u7528\u4ee5\u4e0b\u547d\u4ee4\u6253\u5f00 Windows \u8282\u70b9\u7684 <code>TESTSIGNING</code> \u542f\u52a8\u9879\uff0c\u6267\u884c\u6210\u529f\u540e\u9700\u8981\u91cd\u542f\u7cfb\u7edf\u751f\u6548\uff1a</p> <pre><code>bcdedit /set LOADOPTIONS DISABLE_INTEGRITY_CHECKS\nbcdedit /set TESTSIGNING ON\nbcdedit /set nointegritychecks ON\n</code></pre> <p>\u5728 Windows \u8282\u70b9\u4e0b\u8f7d Windows \u5b89\u88c5\u5305\u5e76\u89e3\u538b\u5b89\u88c5\u3002</p> <p>\u5b89\u88c5\u5b8c\u6210\u540e\u786e\u8ba4\u670d\u52a1\u6b63\u5e38\u8fd0\u884c\uff1a</p> <pre><code>PS &gt; Get-Service | findstr ovs\nRunning  ovsdb-server  Open vSwitch DB Service\nRunning  ovs-vswitchd  Open vSwitch Service\n</code></pre>"},{"location":"advance/windows/#kube-ovn","title":"\u5b89\u88c5 Kube-OVN","text":"<p>\u5728 Windows \u8282\u70b9\u4e0b\u8f7d\u5b89\u88c5\u811a\u672c install.ps1\u3002</p> <p>\u8865\u5145\u76f8\u5173\u53c2\u6570\u5e76\u6267\u884c\uff1a</p> <pre><code>.\\install.ps1 -KubeConfig C:\\k\\admin.conf -ApiServer https://192.168.140.180:6443 -ServiceCIDR 10.96.0.0/12\n</code></pre> <p>\u9ed8\u8ba4\u60c5\u51b5\u4e0b, Kube-OVN \u4f7f\u7528\u8282\u70b9 IP \u6240\u5728\u7684\u7f51\u5361\u4f5c\u4e3a\u96a7\u9053\u63a5\u53e3\u3002 \u5982\u679c\u9700\u8981\u4f7f\u7528\u5176\u5b83\u7f51\u5361\uff0c\u9700\u8981\u5728\u5b89\u88c5\u524d\u7ed9\u8282\u70b9\u6dfb\u52a0\u6307\u5b9a\u7684 Annotation\uff0c\u5982 <code>ovn.kubernetes.io/tunnel_interface=Ethernet1</code>\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/with-bgp/","title":"BGP \u652f\u6301","text":"<p>Kube-OVN \u652f\u6301\u5c06 Pod \u6216 Subnet \u7684 IP \u5730\u5740\u901a\u8fc7 BGP \u534f\u8bae\u5411\u5916\u90e8\u8fdb\u884c\u8def\u7531\u5e7f\u64ad\uff0c\u4ece\u800c\u4f7f\u5f97 Pod IP \u53ef\u4ee5\u76f4\u63a5\u5bf9\u5916\u66b4\u9732\u3002 \u5982\u679c\u9700\u8981\u4f7f\u7528\u8be5\u529f\u80fd\uff0c\u9700\u8981\u5728\u7279\u5b9a\u8282\u70b9\u5b89\u88c5 <code>kube-ovn-speaker</code> \u5e76\u5bf9\u9700\u8981\u5bf9\u5916\u66b4\u9732\u7684 Pod \u6216 Subnet \u589e\u52a0\u5bf9\u5e94\u7684 annotation\u3002</p>"},{"location":"advance/with-bgp/#kube-ovn-speaker","title":"\u5b89\u88c5 kube-ovn-speaker","text":"<p><code>kube-ovn-speaker</code> \u5185\u4f7f\u7528 GoBGP \u5bf9\u5916\u53d1\u5e03\u8def\u7531\u4fe1\u606f\uff0c\u5e76\u5c06\u8bbf\u95ee\u66b4\u9732\u5730\u5740\u7684\u4e0b\u4e00\u8df3\u8def\u7531\u6307\u5411\u81ea\u8eab\u3002</p> <p>\u7531\u4e8e\u90e8\u7f72 <code>kube-ovn-speaker</code> \u7684\u8282\u70b9\u9700\u8981\u627f\u62c5\u56de\u7a0b\u6d41\u91cf\uff0c\u56e0\u6b64\u9700\u8981\u9009\u62e9\u7279\u5b9a\u8282\u70b9\u8fdb\u884c\u90e8\u7f72\uff1a</p> <pre><code>kubectl label nodes speaker-node-1 ovn.kubernetes.io/bgp=true\nkubectl label nodes speaker-node-2 ovn.kubernetes.io/bgp=true\n</code></pre> <p>\u5f53\u5b58\u5728\u591a\u4e2a kube-ovn-speaker \u5b9e\u4f8b\u65f6\uff0c\u6bcf\u4e2a\u5b9e\u4f8b\u90fd\u4f1a\u5bf9\u5916\u53d1\u5e03\u8def\u7531\uff0c\u4e0a\u6e38\u8def\u7531\u5668\u9700\u8981\u652f\u6301\u591a\u8def\u5f84 ECMP\u3002</p> <p>\u4e0b\u8f7d\u5bf9\u5e94 yaml:</p> <pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/yamls/speaker.yaml\n</code></pre> <p>\u4fee\u6539 yaml \u5185\u76f8\u5e94\u914d\u7f6e\uff1a</p> <pre><code>--neighbor-address=10.32.32.1\n--neighbor-as=65030\n--cluster-as=65000\n</code></pre> <ul> <li><code>neighbor-address</code>: BGP Peer \u7684\u5730\u5740\uff0c\u901a\u5e38\u4e3a\u8def\u7531\u5668\u7f51\u5173\u5730\u5740\u3002</li> <li><code>neighbor-as</code>: BGP Peer \u7684 AS \u53f7\u3002</li> <li><code>cluster-as</code>: \u5bb9\u5668\u7f51\u7edc\u7684 AS \u53f7\u3002</li> </ul> <p>\u90e8\u7f72 yaml:</p> <pre><code>kubectl apply -f speaker.yaml\n</code></pre>"},{"location":"advance/with-bgp/#podsubnet","title":"\u53d1\u5e03 Pod/Subnet \u8def\u7531","text":"<p>\u5982\u9700\u4f7f\u7528 BGP \u5bf9\u5916\u53d1\u5e03\u8def\u7531\uff0c\u9996\u5148\u9700\u8981\u5c06\u5bf9\u5e94 Subnet \u7684 <code>natOutgoing</code> \u8bbe\u7f6e\u4e3a <code>false</code>\uff0c\u4f7f\u5f97 Pod IP \u53ef\u4ee5\u76f4\u63a5\u8fdb\u5165\u5e95\u5c42\u7f51\u7edc\u3002</p> <p>\u589e\u52a0 annotation \u5bf9\u5916\u53d1\u5e03\uff1a</p> <pre><code>kubectl annotate pod sample ovn.kubernetes.io/bgp=true\nkubectl annotate subnet ovn-default ovn.kubernetes.io/bgp=true\n</code></pre> <p>\u5220\u9664 annotation \u53d6\u6d88\u53d1\u5e03\uff1a</p> <pre><code>kubectl annotate pod perf-ovn-xzvd4 ovn.kubernetes.io/bgp-\nkubectl annotate subnet ovn-default ovn.kubernetes.io/bgp-\n</code></pre>"},{"location":"advance/with-bgp/#bgp_1","title":"BGP \u9ad8\u7ea7\u9009\u9879","text":"<p><code>kube-ovn-speaker</code> \u652f\u6301\u66f4\u591a BGP \u53c2\u6570\u8fdb\u884c\u9ad8\u7ea7\u914d\u7f6e\uff0c\u7528\u6237\u53ef\u6839\u636e\u81ea\u5df1\u7f51\u7edc\u73af\u5883\u8fdb\u884c\u8c03\u6574\uff1a</p> <ul> <li><code>announce-cluster-ip</code>: \u662f\u5426\u5bf9\u5916\u53d1\u5e03 Service \u8def\u7531\uff0c\u9ed8\u8ba4\u4e3a <code>false</code>\u3002</li> <li><code>auth-password</code>: BGP peer \u7684\u8bbf\u95ee\u5bc6\u7801\u3002</li> <li><code>holdtime</code>: BGP \u90bb\u5c45\u95f4\u7684\u5fc3\u8df3\u63a2\u6d4b\u65f6\u95f4\uff0c\u8d85\u8fc7\u6539\u65f6\u95f4\u6ca1\u6709\u6d88\u606f\u7684\u90bb\u5c45\u5c06\u4f1a\u88ab\u79fb\u9664\uff0c\u9ed8\u8ba4\u4e3a 90 \u79d2\u3002</li> <li><code>graceful-restart</code>: \u662f\u5426\u542f\u7528 BGP Graceful Restart\u3002</li> <li><code>graceful-restart-time</code>: BGP Graceful restart time \u53ef\u53c2\u8003 RFC4724 3\u3002</li> <li><code>graceful-restart-deferral-time</code>: BGP Graceful restart deferral time \u53ef\u53c2\u8003 RFC4724 4.1\u3002</li> <li><code>passivemode</code>: Speaker \u8fd0\u884c\u5728 passive \u6a21\u5f0f\uff0c\u4e0d\u4e3b\u52a8\u8fde\u63a5 peer\u3002</li> <li><code>ebgp-multihop</code>: ebgp ttl \u9ed8\u8ba4\u503c\u4e3a 1\u3002</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/with-cilium/","title":"Cilium \u96c6\u6210","text":"<p>Cilium \u662f\u4e00\u6b3e\u57fa\u4e8e eBPF \u7684\u7f51\u7edc\u548c\u5b89\u5168\u7ec4\u4ef6\uff0cKube-OVN \u5229\u7528\u5176\u4e2d\u7684 CNI Chaining \u6a21\u5f0f\u6765\u5bf9\u5df2\u6709\u529f\u80fd\u8fdb\u884c\u589e\u5f3a\u3002 \u7528\u6237\u53ef\u4ee5\u540c\u65f6\u4f7f\u7528 Kube-OVN \u4e30\u5bcc\u7684\u7f51\u7edc\u62bd\u8c61\u80fd\u529b\u548c eBPF \u5e26\u6765\u7684\u76d1\u63a7\u548c\u5b89\u5168\u80fd\u529b\u3002</p> <p>\u901a\u8fc7\u96c6\u6210 Cilium\uff0cKube-OVN \u7528\u6237\u53ef\u4ee5\u83b7\u5f97\u5982\u4e0b\u589e\u76ca\uff1a</p> <ul> <li>\u66f4\u4e30\u5bcc\u9ad8\u6548\u7684\u5b89\u5168\u7b56\u7565\u3002</li> <li>\u57fa\u4e8e Hubble \u7684\u76d1\u63a7\u89c6\u56fe\u3002</li> </ul> <p></p>"},{"location":"advance/with-cilium/#_1","title":"\u524d\u63d0\u6761\u4ef6","text":"<ol> <li>Linux \u5185\u6838\u7248\u672c\u9ad8\u4e8e 4.19 \u6216\u5176\u4ed6\u517c\u5bb9\u5185\u6838\u4ee5\u83b7\u5f97\u5b8c\u6574 eBPF \u80fd\u529b\u652f\u6301\u3002</li> <li>\u63d0\u524d\u90e8\u7f72 Helm \u4e3a\u5b89\u88c5 Cilium \u505a\u51c6\u5907\uff0c\u90e8\u7f72 Helm \u8bf7\u53c2\u8003 Installing Helm\u3002</li> </ol>"},{"location":"advance/with-cilium/#kube-ovn","title":"\u914d\u7f6e Kube-OVN","text":"<p>\u4e3a\u4e86\u5145\u5206\u4f7f\u7528 Cilium \u7684\u5b89\u5168\u80fd\u529b\uff0c\u9700\u8981\u5173\u95ed Kube-OVN \u5185\u7684 <code>networkpolicy</code> \u529f\u80fd\uff0c\u5e76\u8c03\u6574 CNI \u914d\u7f6e\u4f18\u5148\u7ea7\u3002</p> <p>\u5728 <code>install.sh</code> \u811a\u672c\u91cc\u4fee\u6539\u4e0b\u5217\u53d8\u91cf\uff1a</p> <pre><code>ENABLE_NP=false\nCNI_CONFIG_PRIORITY=10\n</code></pre> <p>\u82e5\u5df2\u90e8\u7f72\u5b8c\u6210\uff0c\u53ef\u901a\u8fc7\u4fee\u6539 <code>kube-ovn-controller</code> \u7684\u542f\u52a8\u53c2\u6570\u8fdb\u884c\u8c03\u6574 <code>networkpolicy</code>\uff1a</p> <pre><code>args:\n- --enable-np=false\n</code></pre> <p>\u4fee\u6539 <code>kube-ovn-cni</code> \u542f\u52a8\u53c2\u6570\u8c03\u6574 CNI \u914d\u7f6e\u4f18\u5148\u7ea7\uff1a</p> <pre><code>args:\n- --cni-conf-name=10-kube-ovn.conflist\n</code></pre> <p>\u5728\u6bcf\u4e2a\u8282\u70b9\u8c03\u6574 Kube-OVN \u914d\u7f6e\u6587\u4ef6\u540d\u79f0\uff0c\u4ee5\u4fbf\u4f18\u5148\u4f7f\u7528 Cilium \u8fdb\u884c\u64cd\u4f5c\uff1a</p> <pre><code>mv /etc/cni/net.d/01-kube-ovn.conflist /etc/cni/net.d/10-kube-ovn.conflist\n</code></pre>"},{"location":"advance/with-cilium/#cilium_1","title":"\u90e8\u7f72 Cilium","text":"<p>\u521b\u5efa <code>chaining.yaml</code> \u914d\u7f6e\u6587\u4ef6\uff0c\u4f7f\u7528 Cilium \u7684 generic-veth \u6a21\u5f0f\uff1a</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: cni-configuration\nnamespace: kube-system\ndata:\ncni-config: |-\n{\n\"name\": \"generic-veth\",\n\"cniVersion\": \"0.3.1\",\n\"plugins\": [\n{\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\",\n\"ipam\": {\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\"\n}\n},\n{\n\"type\": \"portmap\",\n\"snat\": true,\n\"capabilities\": {\"portMappings\": true}\n},\n{\n\"type\": \"cilium-cni\"\n}\n]\n}\n</code></pre> <p>\u5b89\u88c5\u914d\u7f6e\u6587\u4ef6\uff1a</p> <pre><code>kubectl apply -f chaining.yaml\n</code></pre> <p>\u4f7f\u7528 Helm \u90e8\u7f72 Cilium\uff1a</p> <pre><code>helm repo add cilium https://helm.cilium.io/\nhelm install cilium cilium/cilium --version 1.11.6 \\\n--namespace kube-system \\\n--set cni.chainingMode=generic-veth \\\n--set cni.customConf=true \\\n--set cni.configMap=cni-configuration \\\n--set tunnel=disabled \\\n--set enableIPv4Masquerade=false \\\n--set enableIdentityMark=false </code></pre> <p>\u786e\u8ba4 Cilium \u5b89\u88c5\u6210\u529f\uff1a</p> <pre><code># cilium  status\n/\u00af\u00af\\\n/\u00af\u00af\\__/\u00af\u00af\\    Cilium:         OK\n \\__/\u00af\u00af\\__/    Operator:       OK\n /\u00af\u00af\\__/\u00af\u00af\\    Hubble:         disabled\n \\__/\u00af\u00af\\__/    ClusterMesh:    disabled\n    \\__/\n\nDaemonSet         cilium             Desired: 2, Ready: 2/2, Available: 2/2\nDeployment        cilium-operator    Desired: 2, Ready: 2/2, Available: 2/2\nContainers:       cilium             Running: 2\ncilium-operator    Running: 2\nCluster Pods:     8/11 managed by Cilium\nImage versions    cilium             quay.io/cilium/cilium:v1.10.5@sha256:0612218e28288db360c63677c09fafa2d17edda4f13867bcabf87056046b33bb: 2\ncilium-operator    quay.io/cilium/operator-generic:v1.10.5@sha256:2d2f730f219d489ff0702923bf24c0002cd93eb4b47ba344375566202f56d972: 2\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/with-openstack/","title":"OpenStack \u96c6\u6210","text":"<p>\u5728\u4e00\u4e9b\u60c5\u51b5\u4e0b\uff0c\u7528\u6237\u9700\u8981\u4f7f\u7528 OpenStack \u8fd0\u884c\u865a\u62df\u673a\uff0c\u4f7f\u7528 Kubernetes \u8fd0\u884c\u5bb9\u5668\uff0c\u5e76\u9700\u8981\u5bb9\u5668\u548c\u865a\u673a\u4e4b\u95f4\u7f51\u7edc\u4e92\u901a\u5e76\u5904\u4e8e\u7edf\u4e00\u63a7\u5236\u5e73\u9762\u4e0b\u3002\u5982\u679c OpenStack Neutron \u4fa7\u540c\u6837\u4f7f\u7528 OVN \u4f5c\u4e3a\u5e95\u5c42\u7f51\u7edc\u63a7\u5236\uff0c\u90a3\u4e48 Kube-OVN \u53ef\u4ee5\u4f7f\u7528\u96c6\u7fa4\u4e92\u8054\u548c\u5171\u4eab\u5e95\u5c42 OVN \u4e24\u79cd\u65b9\u5f0f\u6253\u901a OpenStack \u548c Kubernetes \u7684\u7f51\u7edc\u3002</p>"},{"location":"advance/with-openstack/#_1","title":"\u96c6\u7fa4\u4e92\u8054","text":"<p>\u8be5\u6a21\u5f0f\u548c\u4f7f\u7528 OVN-IC \u8fdb\u884c\u591a\u96c6\u7fa4\u4e92\u8054\u6253\u901a\u4e24\u4e2a Kubernetes \u96c6\u7fa4\u7f51\u7edc\u65b9\u5f0f\u7c7b\u4f3c\uff0c\u53ea\u4e0d\u8fc7\u5c06\u96c6\u7fa4\u4e24\u7aef\u6362\u6210 OpenStack \u548c Kubernetes\u3002</p>"},{"location":"advance/with-openstack/#_2","title":"\u524d\u63d0\u6761\u4ef6","text":"<ol> <li>\u81ea\u52a8\u4e92\u8054\u6a21\u5f0f\u4e0b OpenStack \u548c Kubernetes \u5185\u7684\u5b50\u7f51 CIDR \u4e0d\u80fd\u76f8\u4e92\u91cd\u53e0\u3002</li> <li>\u9700\u8981\u5b58\u5728\u4e00\u7ec4\u673a\u5668\u53ef\u4ee5\u88ab\u6bcf\u4e2a\u96c6\u7fa4\u901a\u8fc7\u7f51\u7edc\u8bbf\u95ee\uff0c\u7528\u6765\u90e8\u7f72\u8de8\u96c6\u7fa4\u4e92\u8054\u7684\u63a7\u5236\u5668\u3002</li> <li>\u6bcf\u4e2a\u96c6\u7fa4\u9700\u8981\u6709\u4e00\u7ec4\u53ef\u4ee5\u901a\u8fc7 IP \u8fdb\u884c\u8de8\u96c6\u7fa4\u4e92\u8bbf\u7684\u673a\u5668\u4f5c\u4e3a\u4e4b\u540e\u7684\u7f51\u5173\u8282\u70b9\u3002</li> <li>\u8be5\u65b9\u6848\u53ea\u6253\u901a Kubernetes \u9ed8\u8ba4\u5b50\u7f51\u548c OpenStack \u7684\u9009\u5b9a VPC\u3002</li> </ol>"},{"location":"advance/with-openstack/#ovn-ic","title":"\u90e8\u7f72 OVN-IC \u6570\u636e\u5e93","text":"<p>\u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u542f\u52a8 <code>OVN-IC</code> \u6570\u636e\u5e93\uff1a</p> <pre><code>docker run --name=ovn-ic-db -d --network=host -v /etc/ovn/:/etc/ovn -v /var/run/ovn:/var/run/ovn -v /var/log/ovn:/var/log/ovn kubeovn/kube-ovn:v1.13.0 bash start-ic-db.sh\n</code></pre>"},{"location":"advance/with-openstack/#kubernetes","title":"Kubernetes \u4fa7\u64cd\u4f5c","text":"<p>\u5728 <code>kube-system</code> Namespace \u4e0b\u521b\u5efa <code>ovn-ic-config</code> ConfigMap\uff1a</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: ovn-ic-config\nnamespace: kube-system\ndata:\nenable-ic: \"true\"\naz-name: \"az1\" ic-db-host: \"192.168.65.3\"\nic-nb-port: \"6645\" ic-sb-port: \"6646\"\ngw-nodes: \"az1-gw\"\nauto-route: \"true\"\n</code></pre> <ul> <li><code>enable-ic</code>: \u662f\u5426\u5f00\u542f\u96c6\u7fa4\u4e92\u8054\u3002</li> <li><code>az-name</code>: \u533a\u5206\u4e0d\u540c\u96c6\u7fa4\u7684\u96c6\u7fa4\u540d\u79f0\uff0c\u6bcf\u4e2a\u4e92\u8054\u96c6\u7fa4\u9700\u4e0d\u540c\u3002</li> <li><code>ic-db-host</code>: \u90e8\u7f72 <code>OVN-IC</code> \u6570\u636e\u5e93\u7684\u8282\u70b9\u5730\u5740\u3002</li> <li><code>ic-nb-port</code>: <code>OVN-IC</code> \u5317\u5411\u6570\u636e\u5e93\u7aef\u53e3\uff0c\u9ed8\u8ba4\u4e3a 6645\u3002</li> <li><code>ic-sb-port</code>: <code>OVN-IC</code> \u5357\u5411\u6570\u636e\u5e93\u7aef\u53e3\uff0c\u9ed8\u8ba4\u4e3a 6646\u3002</li> <li><code>gw-nodes</code>: \u96c6\u7fa4\u4e92\u8054\u4e2d\u627f\u62c5\u7f51\u5173\u5de5\u4f5c\u7684\u8282\u70b9\u540d\uff0c\u9017\u53f7\u5206\u9694\u3002</li> <li><code>auto-route</code>: \u662f\u5426\u81ea\u52a8\u5bf9\u5916\u53d1\u5e03\u548c\u5b66\u4e60\u8def\u7531\u3002</li> </ul>"},{"location":"advance/with-openstack/#openstack_1","title":"OpenStack \u4fa7\u64cd\u4f5c","text":"<p>\u521b\u5efa\u548c Kubernetes \u4e92\u8054\u7684\u903b\u8f91\u8def\u7531\u5668\uff1a</p> <pre><code># openstack router create router0\n# openstack router list\n+--------------------------------------+---------+--------+-------+----------------------------------+\n| ID                                   | Name    | Status | State | Project                          |\n+--------------------------------------+---------+--------+-------+----------------------------------+\n| d5b38655-249a-4192-8046-71aa4d2b4af1 | router0 | ACTIVE | UP    | 98a29ab7388347e7b5ff8bdd181ba4f9 |\n+--------------------------------------+---------+--------+-------+----------------------------------+\n</code></pre> <p>\u5728 OpenStack \u5185\u7684 OVN \u5317\u5411\u6570\u636e\u5e93\u4e2d\u8bbe\u7f6e\u53ef\u7528\u533a\u540d\u5b57\uff0c\u8be5\u540d\u79f0\u9700\u548c\u5176\u4ed6\u4e92\u8054\u96c6\u7fa4\u4e0d\u540c\uff1a</p> <pre><code>ovn-nbctl set NB_Global . name=op-az\n</code></pre> <p>\u5728\u53ef\u8bbf\u95ee <code>OVN-IC</code> \u6570\u636e\u5e93\u7684\u8282\u70b9\u542f\u52a8 <code>OVN-IC</code> \u63a7\u5236\u5668\uff1a</p> <pre><code>/usr/share/ovn/scripts/ovn-ctl --ovn-ic-nb-db=tcp:192.168.65.3:6645 \\\n--ovn-ic-sb-db=tcp:192.168.65.3:6646 \\\n--ovn-northd-nb-db=unix:/run/ovn/ovnnb_db.sock \\\n--ovn-northd-sb-db=unix:/run/ovn/ovnsb_db.sock \\\nstart_ic\n</code></pre> <ul> <li><code>ovn-ic-nb-db</code>\uff0c<code>ovn-ic-sb-db</code>: OVN-IC \u5317\u5411\u6570\u636e\u5e93\u548c\u5357\u5411\u6570\u636e\u5e93\u5730\u5740\u3002</li> <li><code>ovn-northd-nb-db</code>\uff0c <code>ovn-northd-sb-db</code>: \u5f53\u524d\u96c6\u7fa4 OVN \u5317\u5411\u6570\u636e\u5e93\u548c\u5357\u5411\u6570\u636e\u5730\u5740\u3002</li> </ul> <p>\u914d\u7f6e\u4e92\u8054\u7f51\u5173\u8282\u70b9\uff1a</p> <pre><code>ovs-vsctl set open_vswitch . external_ids:ovn-is-interconn=true\n</code></pre> <p>\u63a5\u4e0b\u6765\u9700\u8981\u5728 OpenStack \u7684 OVN \u5185\u8fdb\u884c\u64cd\u4f5c\u521b\u5efa\u903b\u8f91\u62d3\u6251\u3002</p> <p>\u8fde\u63a5 <code>ts</code> \u4e92\u8054\u4ea4\u6362\u673a\u548c <code>router0</code> \u903b\u8f91\u8def\u7531\u5668\uff0c\u5e76\u8bbe\u7f6e\u76f8\u5173\u89c4\u5219\uff1a</p> <pre><code>ovn-nbctl lrp-add router0 lrp-router0-ts 00:02:ef:11:39:4f 169.254.100.73/24\novn-nbctl lsp-add ts lsp-ts-router0 -- lsp-set-addresses lsp-ts-router0 router \\\n-- lsp-set-type lsp-ts-router0 router \\\n-- lsp-set-options lsp-ts-router0  router-port=lrp-router0-ts\novn-nbctl lrp-set-gateway-chassis lrp-router0-ts {gateway chassis} 1000\novn-nbctl set NB_Global . options:ic-route-adv=true options:ic-route-learn=true\n</code></pre> <p>\u9a8c\u8bc1\u5df2\u5b66\u4e60\u5230 Kubernetes \u8def\u7531\u89c4\u5219\uff1a</p> <pre><code># ovn-nbctl lr-route-list router0\nIPv4 Routes\n                10.0.0.22            169.254.100.34 dst-ip (learned)\n10.16.0.0/16            169.254.100.34 dst-ip (learned)\n</code></pre> <p>\u63a5\u4e0b\u6765\u53ef\u4ee5\u5728 <code>router0</code> \u7f51\u7edc\u4e0b\u521b\u5efa\u865a\u673a\u9a8c\u8bc1\u662f\u5426\u53ef\u4ee5\u548c Kubernetes \u4e0b Pod \u4e92\u901a\u3002</p>"},{"location":"advance/with-openstack/#ovn","title":"\u5171\u4eab\u5e95\u5c42 OVN","text":"<p>\u5728\u8be5\u65b9\u6848\u4e0b\uff0cOpenStack \u548c Kubernetes \u5171\u4eab\u4f7f\u7528\u540c\u4e00\u4e2a OVN\uff0c\u56e0\u6b64\u53ef\u4ee5\u5c06\u4e24\u8005\u7684 VPC \u548c Subnet \u7b49\u6982\u5ff5\u62c9\u9f50\uff0c\u5b9e\u73b0\u66f4\u597d\u7684\u63a7\u5236\u548c\u4e92\u8054\u3002</p> <p>\u5728\u8be5\u6a21\u5f0f\u4e0b\u6211\u4eec\u6b63\u5e38\u4f7f\u7528 Kube-OVN \u90e8\u7f72 OVN\uff0cOpenStack \u4fee\u6539 Neutron \u914d\u7f6e\u5b9e\u73b0\u8fde\u63a5\u540c\u4e00\u4e2a OVN \u6570\u636e\u5e93\u3002OpenStack \u9700\u4f7f\u7528 networking-ovn \u4f5c\u4e3a Neutron \u540e\u7aef\u5b9e\u73b0\u3002</p>"},{"location":"advance/with-openstack/#neutron","title":"Neutron \u914d\u7f6e\u4fee\u6539","text":"<p>\u4fee\u6539 Neutron \u914d\u7f6e\u6587\u4ef6 <code>/etc/neutron/plugins/ml2/ml2_conf.ini</code>\uff1a</p> <pre><code>[ovn]\n...\novn_nb_connection = tcp:[192.168.137.176]:6641,tcp:[192.168.137.177]:6641,tcp:[192.168.137.178]:6641\novn_sb_connection = tcp:[192.168.137.176]:6642,tcp:[192.168.137.177]:6642,tcp:[192.168.137.178]:6642\novn_l3_scheduler = OVN_L3_SCHEDULER\n</code></pre> <ul> <li><code>ovn_nb_connection</code>\uff0c <code>ovn_sb_connection</code>: \u5730\u5740\u9700\u4fee\u6539\u4e3a Kube-OVN \u90e8\u7f72 <code>ovn-central</code> \u8282\u70b9\u7684\u5730\u5740\u3002</li> </ul> <p>\u4fee\u6539\u6bcf\u4e2a\u8282\u70b9\u7684 OVS \u914d\u7f6e\uff1a</p> <pre><code>ovs-vsctl set open . external-ids:ovn-remote=tcp:[192.168.137.176]:6642,tcp:[192.168.137.177]:6642,tcp:[192.168.137.178]:6642\novs-vsctl set open . external-ids:ovn-encap-type=geneve\novs-vsctl set open . external-ids:ovn-encap-ip=192.168.137.200\n</code></pre> <ul> <li><code>external-ids:ovn-remote</code>: \u5730\u5740\u9700\u4fee\u6539\u4e3a Kube-OVN \u90e8\u7f72 <code>ovn-central</code> \u8282\u70b9\u7684\u5730\u5740\u3002</li> <li><code>ovn-encap-ip</code>: \u4fee\u6539\u4e3a\u5f53\u524d\u8282\u70b9\u7684 IP \u5730\u5740\u3002</li> </ul>"},{"location":"advance/with-openstack/#kubernetes-openstack","title":"\u5728 Kubernetes \u4e2d\u4f7f\u7528 OpenStack \u5185\u8d44\u6e90","text":"<p>\u63a5\u4e0b\u6765\u4ecb\u7ecd\u5982\u4f55\u5728 Kubernetes \u4e2d\u67e5\u8be2 OpenStack \u7684\u7f51\u7edc\u8d44\u6e90\u5e76\u5728 OpenStack \u7684\u5b50\u7f51\u4e2d\u521b\u5efa Pod\u3002</p> <p>\u67e5\u8be2 OpenStack \u4e2d\u5df2\u6709\u7684\u7f51\u7edc\u8d44\u6e90\uff0c\u5982\u4e0b\u8d44\u6e90\u5df2\u7ecf\u9884\u5148\u521b\u5efa\u5b8c\u6210\uff1a</p> <pre><code># openstack router list\n+--------------------------------------+---------+--------+-------+----------------------------------+\n| ID                                   | Name    | Status | State | Project                          |\n+--------------------------------------+---------+--------+-------+----------------------------------+\n| 22040ed5-0598-4f77-bffd-e7fd4db47e93 | router0 | ACTIVE | UP    | 62381a21d569404aa236a5dd8712449c |\n+--------------------------------------+---------+--------+-------+----------------------------------+\n# openstack network list\n+--------------------------------------+----------+--------------------------------------+\n| ID                                   | Name     | Subnets                              |\n+--------------------------------------+----------+--------------------------------------+\n| cd59e36a-37db-4c27-b709-d35379a7920f | provider | 01d73d9f-fdaa-426c-9b60-aa34abbfacae |\n+--------------------------------------+----------+--------------------------------------+\n# openstack subnet list\n+--------------------------------------+-------------+--------------------------------------+----------------+\n| ID                                   | Name        | Network                              | Subnet         |\n+--------------------------------------+-------------+--------------------------------------+----------------+\n| 01d73d9f-fdaa-426c-9b60-aa34abbfacae | provider-v4 | cd59e36a-37db-4c27-b709-d35379a7920f | 192.168.1.0/24 |\n+--------------------------------------+-------------+--------------------------------------+----------------+\n# openstack server list\n+--------------------------------------+-------------------+--------+-----------------------+--------+--------+\n| ID                                   | Name              | Status | Networks              | Image  | Flavor |\n+--------------------------------------+-------------------+--------+-----------------------+--------+--------+\n| 8433d622-a8d6-41a7-8b31-49abfd64f639 | provider-instance | ACTIVE | provider=192.168.1.61 | ubuntu | m1     |\n+--------------------------------------+-------------------+--------+-----------------------+--------+--------+\n</code></pre> <p>\u5728 Kubernetes \u4fa7\uff0c\u67e5\u8be2 VPC \u8d44\u6e90\uff1a</p> <pre><code># kubectl get vpc\nNAME                                           STANDBY   SUBNETS\nneutron-22040ed5-0598-4f77-bffd-e7fd4db47e93   true      [\"neutron-cd59e36a-37db-4c27-b709-d35379a7920f\"]\novn-cluster                                    true      [\"join\",\"ovn-default\"]\n</code></pre> <p><code>neutron-22040ed5-0598-4f77-bffd-e7fd4db47e93</code> \u4e3a\u4ece OpenStack \u540c\u6b65\u8fc7\u6765\u7684 VPC \u8d44\u6e90\u3002</p> <p>\u63a5\u4e0b\u6765\u53ef\u4ee5\u6309\u7167 Kube-OVN \u539f\u751f\u7684 VPC \u548c Subnet \u64cd\u4f5c\u521b\u5efa Pod \u5e76\u8fd0\u884c\u3002</p> <p>VPC, Subnet \u7ed1\u5b9a Namespace <code>net2</code>\uff0c\u5e76\u521b\u5efa Pod:</p> <pre><code>apiVersion: v1\nkind: Namespace\nmetadata:\nname: net2\n---\napiVersion: kubeovn.io/v1\nkind: Vpc\nmetadata:\ncreationTimestamp: \"2021-06-20T13:34:11Z\"\ngeneration: 2\nlabels:\novn.kubernetes.io/vpc_external: \"true\"\nname: neutron-22040ed5-0598-4f77-bffd-e7fd4db47e93\nresourceVersion: \"583728\"\nuid: 18d4c654-f511-4def-a3a0-a6434d237c1e\nspec:\nnamespaces:\n- net2\n---\nkind: Subnet\napiVersion: kubeovn.io/v1\nmetadata:\nname: net2\nspec:\nvpc: neutron-22040ed5-0598-4f77-bffd-e7fd4db47e93\nnamespaces:\n- net2\ncidrBlock: 12.0.1.0/24\nnatOutgoing: false\n---\napiVersion: v1\nkind: Pod\nmetadata:\nname: ubuntu\nnamespace: net2\nspec:\ncontainers:\n- image: docker.io/kubeovn/kube-ovn:v1.8.0\ncommand:\n- \"sleep\"\n- \"604800\"\nimagePullPolicy: IfNotPresent\nname: ubuntu\nrestartPolicy: Always\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/with-ovn-ic/","title":"\u4f7f\u7528 OVN-IC \u8fdb\u884c\u591a\u96c6\u7fa4\u4e92\u8054","text":"<p>Kube-OVN \u652f\u6301\u901a\u8fc7 OVN-IC \u5c06\u4e24\u4e2a Kubernetes \u96c6\u7fa4 Pod \u7f51\u7edc\u6253\u901a\uff0c\u6253\u901a\u540e\u7684\u4e24\u4e2a\u96c6\u7fa4\u5185\u7684 Pod \u53ef\u4ee5\u901a\u8fc7 Pod IP \u8fdb\u884c\u76f4\u63a5\u901a\u4fe1\u3002 Kube-OVN \u4f7f\u7528\u96a7\u9053\u5bf9\u8de8\u96c6\u7fa4\u6d41\u91cf\u8fdb\u884c\u5c01\u88c5\uff0c\u4e24\u4e2a\u96c6\u7fa4\u4e4b\u95f4\u53ea\u8981\u5b58\u5728\u4e00\u7ec4 IP \u53ef\u8fbe\u7684\u673a\u5668\u5373\u53ef\u5b8c\u6210\u5bb9\u5668\u7f51\u7edc\u7684\u4e92\u901a\u3002</p> <p>\u8be5\u6a21\u5f0f\u7684\u591a\u96c6\u7fa4\u4e92\u8054\u4e3a Overlay \u7f51\u7edc\u529f\u80fd\uff0cUnderlay \u7f51\u7edc\u5982\u679c\u60f3\u8981\u5b9e\u73b0\u96c6\u7fa4\u4e92\u8054\u9700\u8981\u5e95\u5c42\u57fa\u7840\u8bbe\u65bd\u505a\u7f51\u7edc\u6253\u901a\u3002</p> <p></p>"},{"location":"advance/with-ovn-ic/#_1","title":"\u524d\u63d0\u6761\u4ef6","text":"<ol> <li>\u81ea\u52a8\u4e92\u8054\u6a21\u5f0f\u4e0b\u4e0d\u540c\u96c6\u7fa4\u7684\u5b50\u7f51 CIDR \u4e0d\u80fd\u76f8\u4e92\u91cd\u53e0\uff0c\u9ed8\u8ba4\u5b50\u7f51\u9700\u5728\u5b89\u88c5\u65f6\u914d\u7f6e\u4e3a\u4e0d\u91cd\u53e0\u7684\u7f51\u6bb5\u3002\u82e5\u5b58\u5728\u91cd\u53e0\u9700\u53c2\u8003\u540e\u7eed\u624b\u52a8\u4e92\u8054\u8fc7\u7a0b\uff0c\u53ea\u80fd\u5c06\u4e0d\u91cd\u53e0\u7f51\u6bb5\u6253\u901a\u3002</li> <li>\u9700\u8981\u5b58\u5728\u4e00\u7ec4\u673a\u5668\u53ef\u4ee5\u88ab\u6bcf\u4e2a\u96c6\u7fa4\u7684 <code>kube-ovn-controller</code> \u901a\u8fc7 IP \u8bbf\u95ee\uff0c\u7528\u6765\u90e8\u7f72\u8de8\u96c6\u7fa4\u4e92\u8054\u7684\u63a7\u5236\u5668\u3002</li> <li>\u6bcf\u4e2a\u96c6\u7fa4\u9700\u8981\u6709\u4e00\u7ec4\u53ef\u4ee5\u901a\u8fc7 IP \u8fdb\u884c\u8de8\u96c6\u7fa4\u4e92\u8bbf\u7684\u673a\u5668\u4f5c\u4e3a\u4e4b\u540e\u7684\u7f51\u5173\u8282\u70b9\u3002</li> <li>\u8be5\u529f\u80fd\u53ea\u5bf9\u9ed8\u8ba4 VPC \u751f\u6548\uff0c\u7528\u6237\u81ea\u5b9a\u4e49 VPC \u65e0\u6cd5\u4f7f\u7528\u4e92\u8054\u529f\u80fd\u3002</li> </ol>"},{"location":"advance/with-ovn-ic/#ovn-ic_1","title":"\u90e8\u7f72\u5355\u8282\u70b9 OVN-IC \u6570\u636e\u5e93","text":"<p>\u5728\u6bcf\u4e2a\u96c6\u7fa4 <code>kube-ovn-controller</code> \u53ef\u901a\u8fc7 IP \u8bbf\u95ee\u7684\u673a\u5668\u4e0a\u90e8\u7f72 <code>OVN-IC</code> \u6570\u636e\u5e93\uff0c\u8be5\u8282\u70b9\u5c06\u4fdd\u5b58\u5404\u4e2a\u96c6\u7fa4\u540c\u6b65\u4e0a\u6765\u7684\u7f51\u7edc\u914d\u7f6e\u4fe1\u606f\u3002</p> <p>\u90e8\u7f72 <code>docker</code> \u7684\u73af\u5883\u53ef\u4ee5\u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u542f\u52a8 <code>OVN-IC</code> \u6570\u636e\u5e93\uff1a</p> <pre><code>docker run --name=ovn-ic-db -d --network=host --privileged  -v /etc/ovn/:/etc/ovn -v /var/run/ovn:/var/run/ovn -v /var/log/ovn:/var/log/ovn kubeovn/kube-ovn:v1.13.0 bash start-ic-db.sh\n</code></pre> <p>\u5bf9\u4e8e\u90e8\u7f72 <code>containerd</code> \u53d6\u4ee3 <code>docker</code> \u7684\u73af\u5883\u53ef\u4ee5\u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\uff1a</p> <pre><code>ctr -n k8s.io run -d --net-host --privileged --mount=\"type=bind,src=/etc/ovn/,dst=/etc/ovn,options=rbind:rw\" --mount=\"type=bind,src=/var/run/ovn,dst=/var/run/ovn,options=rbind:rw\" --mount=\"type=bind,src=/var/log/ovn,dst=/var/log/ovn,options=rbind:rw\" docker.io/kubeovn/kube-ovn:v1.13.0 ovn-ic-db bash start-ic-db.sh\n</code></pre>"},{"location":"advance/with-ovn-ic/#_2","title":"\u81ea\u52a8\u8def\u7531\u8bbe\u7f6e","text":"<p>\u5728\u81ea\u52a8\u8def\u7531\u8bbe\u7f6e\u4e0b\uff0c\u6bcf\u4e2a\u96c6\u7fa4\u4f1a\u5c06\u81ea\u5df1\u9ed8\u8ba4 VPC \u4e0b Subnet \u7684 CIDR \u4fe1\u606f\u540c\u6b65\u7ed9 <code>OVN-IC</code>\uff0c\u56e0\u6b64\u8981\u786e\u4fdd\u4e24\u4e2a\u96c6\u7fa4\u7684 Subnet CIDR \u4e0d\u5b58\u5728\u91cd\u53e0\u3002</p> <p>\u5728 <code>kube-system</code> Namespace \u4e0b\u521b\u5efa <code>ovn-ic-config</code> ConfigMap\uff1a</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: ovn-ic-config\nnamespace: kube-system\ndata:\nenable-ic: \"true\"\naz-name: \"az1\" ic-db-host: \"192.168.65.3\"\nic-nb-port: \"6645\" ic-sb-port: \"6646\"\ngw-nodes: \"az1-gw\"\nauto-route: \"true\"\n</code></pre> <ul> <li><code>enable-ic</code>: \u662f\u5426\u5f00\u542f\u96c6\u7fa4\u4e92\u8054\u3002</li> <li><code>az-name</code>: \u533a\u5206\u4e0d\u540c\u96c6\u7fa4\u7684\u96c6\u7fa4\u540d\u79f0\uff0c\u6bcf\u4e2a\u4e92\u8054\u96c6\u7fa4\u9700\u4e0d\u540c\u3002</li> <li><code>ic-db-host</code>: \u90e8\u7f72 <code>OVN-IC</code> \u6570\u636e\u5e93\u7684\u8282\u70b9\u5730\u5740\u3002</li> <li><code>ic-nb-port</code>: <code>OVN-IC</code> \u5317\u5411\u6570\u636e\u5e93\u7aef\u53e3\uff0c\u9ed8\u8ba4\u4e3a 6645\u3002</li> <li><code>ic-sb-port</code>: <code>OVN-IC</code> \u5357\u5411\u6570\u636e\u5e93\u7aef\u53e3\uff0c\u9ed8\u8ba4\u4e3a 6646\u3002</li> <li><code>gw-nodes</code>: \u96c6\u7fa4\u4e92\u8054\u4e2d\u627f\u62c5\u7f51\u5173\u5de5\u4f5c\u7684\u8282\u70b9\u540d\uff0c\u9017\u53f7\u5206\u9694\u3002</li> <li><code>auto-route</code>: \u662f\u5426\u81ea\u52a8\u5bf9\u5916\u53d1\u5e03\u548c\u5b66\u4e60\u8def\u7531\u3002</li> </ul> <p>\u6ce8\u610f\uff1a \u4e3a\u4e86\u4fdd\u8bc1\u64cd\u4f5c\u7684\u6b63\u786e\u6027\uff0c<code>ovn-ic-config</code> \u8fd9\u4e2a ConfigMap \u4e0d\u5141\u8bb8\u4fee\u6539\u3002\u5982\u6709\u53c2\u6570\u9700\u8981\u53d8\u66f4\uff0c\u8bf7\u5220\u9664\u8be5 ConfigMap\uff0c\u4fee\u6539\u540e\u518d\u5e94\u7528\u6b64 ConfigMap\u3002</p> <p>\u5728 <code>ovn-ic</code> \u5bb9\u5668\u5185\u901a\u8fc7\u4e0b\u9762\u7684\u547d\u4ee4\u67e5\u770b\u662f\u5426\u5df2\u5efa\u7acb\u4e92\u8054\u903b\u8f91\u4ea4\u6362\u673a <code>ts</code>\uff1a</p> <pre><code># ovn-ic-sbctl show\navailability-zone az1\n    gateway deee03e0-af16-4f45-91e9-b50c3960f809\n        hostname: az1-gw\n        type: geneve\n            ip: 192.168.42.145\n        port ts-az1\n            transit switch: ts\n            address: [\"00:00:00:50:AC:8C 169.254.100.45/24\"]\navailability-zone az2\n    gateway e94cc831-8143-40e3-a478-90352773327b\n        hostname: az2-gw\n        type: geneve\n            ip: 192.168.42.149\n        port ts-az2\n            transit switch: ts\n            address: [\"00:00:00:07:4A:59 169.254.100.63/24\"]\n</code></pre> <p>\u5728\u6bcf\u4e2a\u96c6\u7fa4\u89c2\u5bdf\u903b\u8f91\u8def\u7531\u662f\u5426\u6709\u5b66\u4e60\u5230\u7684\u5bf9\u7aef\u8def\u7531\uff1a</p> <pre><code># kubectl ko nbctl lr-route-list ovn-cluster\nIPv4 Routes\n                10.42.1.1            169.254.100.45 dst-ip (learned)\n10.42.1.3                100.64.0.2 dst-ip\n                10.16.0.2                100.64.0.2 src-ip\n                10.16.0.3                100.64.0.2 src-ip\n                10.16.0.4                100.64.0.2 src-ip\n                10.16.0.6                100.64.0.2 src-ip\n             10.17.0.0/16            169.254.100.45 dst-ip (learned)\n100.65.0.0/16            169.254.100.45 dst-ip (learned)\n</code></pre> <p>\u63a5\u4e0b\u6765\u53ef\u4ee5\u5c1d\u8bd5\u5728\u96c6\u7fa4 1 \u5185\u7684\u4e00\u4e2a Pod \u5185\u76f4\u63a5 <code>ping</code> \u96c6\u7fa4 2 \u5185\u7684\u4e00\u4e2a Pod IP \u89c2\u5bdf\u662f\u5426\u53ef\u4ee5\u8054\u901a\u3002</p> <p>\u5bf9\u4e8e\u67d0\u4e2a\u4e0d\u60f3\u5bf9\u5916\u81ea\u52a8\u53d1\u5e03\u8def\u7531\u7684\u5b50\u7f51\u53ef\u4ee5\u901a\u8fc7\u4fee\u6539 Subnet \u91cc\u7684 <code>disableInterConnection</code> \u6765\u7981\u6b62\u8def\u7531\u5e7f\u64ad\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: no-advertise\nspec:\ncidrBlock: 10.199.0.0/16\ndisableInterConnection: true\n</code></pre>"},{"location":"advance/with-ovn-ic/#_3","title":"\u624b\u52a8\u8def\u7531\u8bbe\u7f6e","text":"<p>\u5bf9\u4e8e\u96c6\u7fa4\u95f4\u5b58\u5728\u91cd\u53e0 CIDR \u53ea\u5e0c\u671b\u505a\u90e8\u5206\u5b50\u7f51\u6253\u901a\u7684\u60c5\u51b5\uff0c\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u7684\u6b65\u9aa4\u624b\u52a8\u53d1\u5e03\u5b50\u7f51\u8def\u7531\u3002</p> <p>\u5728 <code>kube-system</code> Namespace \u4e0b\u521b\u5efa <code>ovn-ic-config</code> ConfigMap\uff0c\u5e76\u5c06 <code>auto-route</code> \u8bbe\u7f6e\u4e3a <code>false</code>\uff1a</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: ovn-ic-config\nnamespace: kube-system\ndata:\nenable-ic: \"true\"\naz-name: \"az1\" ic-db-host: \"192.168.65.3\"\nic-nb-port: \"6645\" ic-sb-port: \"6646\"\ngw-nodes: \"az1-gw\"\nauto-route: \"false\"\n</code></pre> <p>\u5728\u6bcf\u4e2a\u96c6\u7fa4\u5206\u522b\u67e5\u770b\u8fdc\u7aef\u903b\u8f91\u7aef\u53e3\u7684\u5730\u5740\uff0c\u7528\u4e8e\u4e4b\u540e\u624b\u52a8\u914d\u7f6e\u8def\u7531\uff1a</p> <pre><code>[root@az1 ~]# kubectl ko nbctl show\nswitch a391d3a1-14a0-4841-9836-4bd930c447fb (ts)\nport ts-az1\n        type: router\n        router-port: az1-ts\n    port ts-az2\n        type: remote\n        addresses: [\"00:00:00:4B:E2:9F 169.254.100.31/24\"]\n\n[root@az2 ~]# kubectl ko nbctl show\nswitch da6138b8-de81-4908-abf9-b2224ec4edf3 (ts)\nport ts-az2\n        type: router\n        router-port: az2-ts\n    port ts-az1\n        type: remote\n        addresses: [\"00:00:00:FB:2A:F7 169.254.100.79/24\"]        </code></pre> <p>\u7531\u4e0a\u8f93\u51fa\u53ef\u77e5\uff0c\u96c6\u7fa4 <code>az1</code> \u5230 \u96c6\u7fa4 <code>az2</code> \u7684\u8fdc\u7aef\u5730\u5740\u4e3a <code>169.254.100.31</code>\uff0c<code>az2</code> \u5230 <code>az1</code> \u7684\u8fdc\u7aef\u5730\u5740\u4e3a <code>169.254.100.79</code>\u3002</p> <p>\u4e0b\u9762\u624b\u52a8\u8bbe\u7f6e\u8def\u7531\uff0c\u5728\u8be5\u4f8b\u5b50\u4e2d\uff0c\u96c6\u7fa4 <code>az1</code> \u5185\u7684\u5b50\u7f51 CIDR \u4e3a <code>10.16.0.0/24</code>\uff0c\u96c6\u7fa4 <code>az2</code> \u5185\u7684\u5b50\u7f51 CIDR \u4e3a <code>10.17.0.0/24</code>\u3002</p> <p>\u5728\u96c6\u7fa4 <code>az1</code> \u8bbe\u7f6e\u5230\u96c6\u7fa4 <code>az2</code> \u7684\u8def\u7531:</p> <pre><code>kubectl ko nbctl lr-route-add ovn-cluster 10.17.0.0/24 169.254.100.31\n</code></pre> <p>\u5728\u96c6\u7fa4 <code>az2</code> \u8bbe\u7f6e\u5230\u96c6\u7fa4 <code>az1</code> \u7684\u8def\u7531:</p> <pre><code>kubectl ko nbctl lr-route-add ovn-cluster 10.16.0.0/24 169.254.100.79\n</code></pre>"},{"location":"advance/with-ovn-ic/#ovn-ic_2","title":"\u9ad8\u53ef\u7528 OVN-IC \u6570\u636e\u5e93\u90e8\u7f72","text":"<p><code>OVN-IC</code> \u6570\u636e\u5e93\u4e4b\u95f4\u53ef\u4ee5\u901a\u8fc7 Raft \u534f\u8bae\u7ec4\u6210\u4e00\u4e2a\u9ad8\u53ef\u7528\u96c6\u7fa4\uff0c\u8be5\u90e8\u7f72\u6a21\u5f0f\u9700\u8981\u81f3\u5c11 3 \u4e2a\u8282\u70b9\u3002</p> <p>\u9996\u5148\u5728\u7b2c\u4e00\u4e2a\u8282\u70b9\u4e0a\u542f\u52a8 <code>OVN-IC</code> \u6570\u636e\u5e93\u7684 leader\u3002</p> <p>\u90e8\u7f72 <code>docker</code> \u73af\u5883\u7684\u7528\u6237\u53ef\u4ee5\u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\uff1a</p> <pre><code>docker run --name=ovn-ic-db -d --network=host --privileged -v /etc/ovn/:/etc/ovn -v /var/run/ovn:/var/run/ovn -v /var/log/ovn:/var/log/ovn -e LOCAL_IP=\"192.168.65.3\"  -e NODE_IPS=\"192.168.65.3,192.168.65.2,192.168.65.1\"   kubeovn/kube-ovn:v1.13.0 bash start-ic-db.sh\n</code></pre> <p>\u5982\u679c\u662f\u90e8\u7f72 <code>containerd</code> \u7684\u7528\u6237\u53ef\u4ee5\u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\uff1a</p> <pre><code>ctr -n k8s.io run -d --net-host --privileged --mount=\"type=bind,src=/etc/ovn/,dst=/etc/ovn,options=rbind:rw\" --mount=\"type=bind,src=/var/run/ovn,dst=/var/run/ovn,options=rbind:rw\" --mount=\"type=bind,src=/var/log/ovn,dst=/var/log/ovn,options=rbind:rw\"  --env=\"NODE_IPS=\"192.168.65.3,192.168.65.2,192.168.65.1\"\" --env=\"LOCAL_IP=\"192.168.65.3\"\" docker.io/kubeovn/kube-ovn:v1.13.0 ovn-ic-db bash start-ic-db.sh\n</code></pre> <ul> <li><code>LOCAL_IP</code>\uff1a \u5f53\u524d\u5bb9\u5668\u6240\u5728\u8282\u70b9 IP \u5730\u5740\u3002</li> <li><code>NODE_IPS</code>\uff1a \u8fd0\u884c <code>OVN-IC</code> \u6570\u636e\u5e93\u7684\u4e09\u4e2a\u8282\u70b9 IP \u5730\u5740\uff0c\u4f7f\u7528\u9017\u53f7\u8fdb\u884c\u5206\u9694\u3002</li> </ul> <p>\u63a5\u4e0b\u6765\uff0c\u5728\u53e6\u5916\u4e24\u4e2a\u8282\u70b9\u90e8\u7f72 <code>OVN-IC</code> \u6570\u636e\u5e93\u7684 follower\u3002</p> <p>\u90e8\u7f72 <code>docker</code> \u73af\u5883\u7684\u7528\u6237\u53ef\u4ee5\u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\uff1a</p> <pre><code>docker run --name=ovn-ic-db -d --network=host --privileged -v /etc/ovn/:/etc/ovn -v /var/run/ovn:/var/run/ovn -v /var/log/ovn:/var/log/ovn -e LOCAL_IP=\"192.168.65.2\"  -e NODE_IPS=\"192.168.65.3,192.168.65.2,192.168.65.1\" -e LEADER_IP=\"192.168.65.3\"  kubeovn/kube-ovn:v1.13.0 bash start-ic-db.sh\n</code></pre> <p>\u5982\u679c\u662f\u90e8\u7f72 <code>containerd</code> \u7684\u7528\u6237\u53ef\u4ee5\u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\uff1a</p> <pre><code>ctr -n k8s.io run -d --net-host --privileged --mount=\"type=bind,src=/etc/ovn/,dst=/etc/ovn,options=rbind:rw\" --mount=\"type=bind,src=/var/run/ovn,dst=/var/run/ovn,options=rbind:rw\" --mount=\"type=bind,src=/var/log/ovn,dst=/var/log/ovn,options=rbind:rw\"  --env=\"NODE_IPS=\"192.168.65.3,192.168.65.2,192.168.65.1\"\" --env=\"LOCAL_IP=\"192.168.65.2\"\" --env=\"LEADER_IP=\"192.168.65.3\"\" docker.io/kubeovn/kube-ovn:v1.13.0 ovn-ic-db bash start-ic-db.sh\n</code></pre> <ul> <li><code>LOCAL_IP</code>\uff1a \u5f53\u524d\u5bb9\u5668\u6240\u5728\u8282\u70b9 IP \u5730\u5740\u3002</li> <li><code>NODE_IPS</code>\uff1a \u8fd0\u884c <code>OVN-IC</code> \u6570\u636e\u5e93\u7684\u4e09\u4e2a\u8282\u70b9 IP \u5730\u5740\uff0c\u4f7f\u7528\u9017\u53f7\u8fdb\u884c\u5206\u9694\u3002</li> <li><code>LEADER_IP</code>: \u8fd0\u884c <code>OVN-IC</code> \u6570\u636e\u5e93 leader \u8282\u70b9\u7684 IP \u5730\u5740\u3002</li> </ul> <p>\u5728\u6bcf\u4e2a\u96c6\u7fa4\u521b\u5efa <code>ovn-ic-config</code> \u65f6\u6307\u5b9a\u591a\u4e2a <code>OVN-IC</code> \u6570\u636e\u5e93\u8282\u70b9\u5730\u5740\uff1a</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: ovn-ic-config\nnamespace: kube-system\ndata:\nenable-ic: \"true\"\naz-name: \"az1\" ic-db-host: \"192.168.65.3,192.168.65.2,192.168.65.1\"\nic-nb-port: \"6645\"\nic-sb-port: \"6646\"\ngw-nodes: \"az1-gw\"\nauto-route: \"true\"\n</code></pre>"},{"location":"advance/with-ovn-ic/#_4","title":"\u624b\u52a8\u91cd\u7f6e","text":"<p>\u5728\u4e00\u4e9b\u60c5\u51b5\u4e0b\uff0c\u7531\u4e8e\u914d\u7f6e\u9519\u8bef\u9700\u8981\u5bf9\u6574\u4e2a\u4e92\u8054\u914d\u7f6e\u8fdb\u884c\u6e05\u7406\uff0c\u53ef\u4ee5\u53c2\u8003\u4e0b\u9762\u7684\u6b65\u9aa4\u6e05\u7406\u73af\u5883\u3002</p> <p>\u5220\u9664\u5f53\u524d\u7684 <code>ovn-ic-config</code> Configmap\uff1a</p> <pre><code>kubectl -n kube-system delete cm ovn-ic-config\n</code></pre> <p>\u5220\u9664 <code>ts</code> \u903b\u8f91\u4ea4\u6362\u673a\uff1a</p> <pre><code>kubectl-ko nbctl ls-del ts\n</code></pre> <p>\u5728\u5bf9\u7aef\u96c6\u7fa4\u91cd\u590d\u540c\u6837\u7684\u6b65\u9aa4\u3002</p>"},{"location":"advance/with-ovn-ic/#az-name","title":"\u4fee\u6539 az-name","text":"<p>\u53ef\u4ee5\u76f4\u63a5\u901a\u8fc7 <code>kubectl edit</code> \u7684\u65b9\u5f0f\u5bf9 <code>ovn-ic-config</code> \u8fd9\u4e2a configmap \u4e2d\u7684 <code>az-name</code> \u5b57\u6bb5\u8fdb\u884c\u4fee\u6539\u3002 \u4f46\u662f\u9700\u8981\u5728\u6bcf\u4e2a ovn-cni pod \u4e0a\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u5426\u5219\u53ef\u80fd\u51fa\u73b0\u6700\u957f 10 \u5206\u949f\u7684\u8de8\u96c6\u7fa4\u7f51\u7edc\u4e2d\u65ad\u3002</p> <pre><code>ovn-appctl -t ovn-controller inc-engine/recompute\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"advance/with-submariner/","title":"\u4f7f\u7528 Submariner \u8fdb\u884c\u591a\u96c6\u7fa4\u4e92\u8054","text":"<p>Submariner \u4f5c\u4e3a\u53ef\u4ee5\u6253\u901a\u591a\u4e2a Kubernetes \u96c6\u7fa4 Pod \u548c Service \u7f51\u7edc\u7684\u5f00\u6e90\u7f51\u7edc\u7ec4\u4ef6\uff0c\u80fd\u591f\u5e2e\u52a9  Kube-OVN \u5b9e\u73b0\u591a\u96c6\u7fa4\u4e92\u8054\u3002</p> <p>\u76f8\u6bd4\u901a\u8fc7 OVN-IC \u6253\u901a\u591a\u96c6\u7fa4\u7f51\u7edc\u7684\u65b9\u5f0f\uff0cSubmariner \u53ef\u4ee5\u6253\u901a Kube-OVN \u548c\u975e Kube-OVN \u7684\u96c6\u7fa4\u7f51\u7edc\uff0c\u5e76 \u80fd\u63d0\u4f9b Service \u7684\u8de8\u96c6\u7fa4\u80fd\u529b\u3002\u4f46\u662f Submariner \u76ee\u524d\u53ea\u80fd\u5b9e\u73b0\u9ed8\u8ba4\u5b50\u7f51\u7684\u6253\u901a\uff0c\u65e0\u6cd5\u5b9e\u73b0\u591a\u5b50\u7f51\u9009\u62e9\u6027\u6253\u901a\u3002</p>"},{"location":"advance/with-submariner/#_1","title":"\u524d\u63d0\u6761\u4ef6","text":"<ul> <li>\u4e24\u4e2a\u96c6\u7fa4\u7684 Service CIDR \u548c\u9ed8\u8ba4\u5b50\u7f51\u7684 CIDR \u4e0d\u80fd\u91cd\u53e0\u3002</li> </ul>"},{"location":"advance/with-submariner/#submariner_1","title":"\u90e8\u7f72 Submariner","text":"<p>\u4e0b\u8f7d <code>subctl</code> \u4e8c\u8fdb\u5236\u6587\u4ef6\uff0c\u5e76\u90e8\u7f72\u5230\u76f8\u5e94\u8def\u5f84\uff1a</p> <pre><code>curl -Ls https://get.submariner.io | bash\nexport PATH=$PATH:~/.local/bin\necho export PATH=\\$PATH:~/.local/bin &gt;&gt; ~/.profile\n</code></pre> <p>\u5207\u6362 <code>kubeconfig</code> \u81f3\u5e0c\u671b\u90e8\u7f72 <code>submariner-broker</code> \u7684\u96c6\u7fa4\u8fdb\u884c\u90e8\u7f72\uff1a</p> <pre><code>subctl deploy-broker\n</code></pre> <p>\u5728\u672c\u6587\u6863\u4e2d <code>cluster0</code> \u7684\u9ed8\u8ba4\u5b50\u7f51 CIDR \u4e3a <code>10.16.0.0/16</code>\uff0cjoin \u5b50\u7f51 CIDR \u4e3a <code>100.64.0.0/16</code>\uff0c<code>cluster1</code> \u7684\u9ed8\u8ba4\u5b50\u7f51 CIDR \u4e3a <code>11.16.0.0/16</code>\uff0cjoin \u5b50\u7f51 CIDR \u4e3a '100.68.0.0/16'\u3002</p> <p>\u5207\u6362 <code>kubeconfig</code> \u81f3 <code>cluster0</code> \u6ce8\u518c\u96c6\u7fa4\u81f3 broker\uff0c\u5e76\u6ce8\u518c\u7f51\u5173\u8282\u70b9:</p> <pre><code>subctl  join broker-info.subm --clusterid  cluster0 --clustercidr 100.64.0.0/16,10.16.0.0/16  --natt=false --cable-driver vxlan --health-check=false\nkubectl label nodes cluster0 submariner.io/gateway=true\n</code></pre> <p>\u5207\u6362 <code>kubeconfig</code> \u81f3 <code>cluster1</code> \u6ce8\u518c\u96c6\u7fa4\u81f3 broker\uff0c\u5e76\u6ce8\u518c\u7f51\u5173\u8282\u70b9:</p> <pre><code>subctl  join broker-info.subm --clusterid  cluster1 --clustercidr 100.68.0.0/16,11.16.0.0/16  --natt=false --cable-driver vxlan --health-check=false\nkubectl label nodes cluster1 submariner.io/gateway=true\n</code></pre> <p>\u5982\u679c\u6267\u884c<code>join</code>\u547d\u4ee4\u4e4b\u540e\u6ca1\u6709\u65b0\u7684<code>gateway, routeagent</code>pod \u51fa\u73b0\u7684\u8bdd, \u8bf7\u4e3a<code>submariner-operator</code>\u8fd9\u4e2a<code>clusterrole</code>\u589e\u52a0\u4ee5\u4e0b\u6743\u9650:</p> <pre><code>- apiGroups:\n- \"apps\"\nresources:\n- daemonsets\nverbs:\n- create\n- get\n- list\n- watch\n- update\n</code></pre> <p>\u5bf9\u4e8e\u591a\u8282\u70b9\u7684\u96c6\u7fa4\uff0c\u9700\u8981\u5c06\u9ed8\u8ba4\u7684<code>subnet</code> <code>ovn-default</code>\u7684\u7f51\u5173\u914d\u7f6e\u6539\u4e3a<code>centralized</code>\u3002\u4e3a submariner \u914d\u7f6e\u7684<code>gateway</code>\u8282\u70b9\u9700\u8981\u548c<code>subnet</code>\u8282\u70b9\u5b8c\u5168\u76f8\u540c\u3002</p> <p>\u63a5\u4e0b\u6765\u53ef\u4ee5\u5728\u4e24\u4e2a\u96c6\u7fa4\u5185\u5206\u522b\u542f\u52a8 Pod \u5e76\u5c1d\u8bd5\u4f7f\u7528 IP \u8fdb\u884c\u76f8\u4e92\u8bbf\u95ee\u3002</p> <p>\u5982\u679c\u51fa\u73b0\u7f51\u7edc\u4e92\u901a\u95ee\u9898\u53ef\u901a\u8fc7 <code>subctl</code> \u547d\u4ee4\u8fdb\u884c\u8bca\u65ad\uff1a</p> <pre><code>subctl show all\nsubctl diagnose all\n</code></pre> <p>\u66f4\u591a Submariner \u76f8\u5173\u64cd\u4f5c\u8bf7\u67e5\u770b Submariner \u7528\u6237\u624b\u518c\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/custom-routes/","title":"\u81ea\u5b9a\u4e49\u8def\u7531","text":"<p>\u53ef\u4ee5\u5728\u521b\u5efa Pod \u65f6\u901a\u8fc7 Annotations \u6765\u6307\u5b9a\u9700\u8981\u914d\u7f6e\u7684\u8def\u7531\uff0c\u5982\u4e0b\u6240\u793a\uff1a</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: custom-routes\nannotations:\novn.kubernetes.io/routes: |\n[{\n\"dst\": \"192.168.0.101/24\",\n\"gw\": \"10.16.0.254\"\n}, {\n\"gw\": \"10.16.0.254\"\n}]\nspec:\ncontainers:\n- name: nginx\nimage: docker.io/library/nginx:alpine\n</code></pre> <p><code>dst</code> \u5b57\u6bb5\u4e3a\u7a7a\u8868\u793a\u4fee\u6539\u9ed8\u8ba4\u8def\u7531\u3002</p> <p>\u5982\u679c\u5de5\u4f5c\u8d1f\u8f7d\u4e3a Deployment\u3001DaemonSet \u6216 StatefulSet\uff0c\u5bf9\u5e94\u7684 Annotation \u9700\u8981\u914d\u7f6e\u5728\u8d44\u6e90\u7684 <code>.spec.template.metadata.annotations</code> \u4e2d\uff0c\u793a\u4f8b\u5982\u4e0b\uff1a</p> <pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nname: custom-routes\nlabels:\napp: nginx\nspec:\nreplicas: 2\nselector:\nmatchLabels:\napp: nginx\ntemplate:\nmetadata:\nlabels:\napp: nginx\nannotations:\novn.kubernetes.io/routes: |\n[{\n\"dst\": \"192.168.0.101/24\",\n\"gw\": \"10.16.0.254\"\n}, {\n\"gw\": \"10.16.0.254\"\n}]\nspec:\ncontainers:\n- name: nginx\nimage: docker.io/library/nginx:alpine\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/dual-stack/","title":"\u53cc\u6808\u4f7f\u7528","text":"<p>Kube-OVN \u4e2d\u4e0d\u540c\u7684\u5b50\u7f51\u53ef\u4ee5\u652f\u6301\u4e0d\u540c\u7684\u534f\u8bae\uff0c\u4e00\u4e2a\u96c6\u7fa4\u5185\u53ef\u4ee5\u540c\u65f6\u5b58\u5728 IPv4\uff0cIPv6 \u548c\u53cc\u6808\u7c7b\u578b\u7684\u5b50\u7f51\u3002 \u6211\u4eec\u63a8\u8350\u4e00\u4e2a\u96c6\u7fa4\u5185\u4f7f\u7528\u7edf\u4e00\u7684\u534f\u8bae\u7c7b\u578b\u4ee5\u7b80\u5316\u4f7f\u7528\u548c\u7ef4\u62a4\u3002</p> <p>\u4e3a\u4e86\u652f\u6301\u53cc\u6808\uff0c\u9700\u8981\u4e3b\u673a\u7f51\u7edc\u6ee1\u8db3\u53cc\u6808\u8981\u6c42\uff0c\u540c\u65f6\u9700\u8981\u5bf9 Kubernetes \u76f8\u5173\u53c2\u6570\u505a\u8c03\u6574\uff0c \u8bf7\u53c2\u8003 Kubernetes \u7684\u53cc\u6808\u5b98\u65b9\u6307\u5bfc\u3002</p>"},{"location":"guide/dual-stack/#_2","title":"\u521b\u5efa\u53cc\u6808\u5b50\u7f51","text":"<p>\u5728\u914d\u7f6e\u53cc\u6808\u65f6\uff0c\u53ea\u9700\u8981\u8bbe\u7f6e\u5bf9\u5e94\u5b50\u7f51 CIDR \u683c\u5f0f\u4e3a <code>cidr=&lt;IPv4 CIDR&gt;,&lt;IPv6 CIDR&gt;</code> \u5373\u53ef\u3002 CIDR \u987a\u5e8f\u8981\u6c42 IPv4 \u5728\u524d\uff0cIPv6 \u5728\u540e\uff0c\u5982\u4e0b\u6240\u793a\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata: name: ovn-test\nspec:\ncidrBlock: 10.16.0.0/16,fd00:10:16::/64\nexcludeIps:\n- 10.16.0.1\n- fd00:10:16::1\ngateway: 10.16.0.1,fd00:10:16::1\n</code></pre> <p>\u5982\u679c\u9700\u8981\u5728\u5b89\u88c5\u65f6\u9ed8\u8ba4\u5b50\u7f51\u4f7f\u7528\u53cc\u6808\uff0c\u9700\u8981\u5728\u5b89\u88c5\u811a\u672c\u4e2d\u4fee\u6539\u5982\u4e0b\u53c2\u6570\uff1a</p> <pre><code>POD_CIDR=\"10.16.0.0/16,fd00:10:16::/64\"\nJOIN_CIDR=\"100.64.0.0/16,fd00:100:64::/64\"\n</code></pre>"},{"location":"guide/dual-stack/#pod","title":"\u67e5\u770b Pod \u5730\u5740","text":"<p>\u914d\u7f6e\u53cc\u6808\u7f51\u7edc\u7684 Pod \u5c06\u4f1a\u4ece\u8be5\u5b50\u7f51\u540c\u65f6\u5206\u914d IPv4 \u548c IPv6 \u7684\u5730\u5740\uff0c\u5206\u914d\u7ed3\u679c\u4f1a\u663e\u793a\u5728 Pod \u7684 annotation \u4e2d:</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nannotations:\novn.kubernetes.io/allocated: \"true\"\novn.kubernetes.io/cidr: 10.16.0.0/16,fd00:10:16::/64\novn.kubernetes.io/gateway: 10.16.0.1,fd00:10:16::1\novn.kubernetes.io/ip_address: 10.16.0.9,fd00:10:16::9\novn.kubernetes.io/logical_switch: ovn-default\novn.kubernetes.io/mac_address: 00:00:00:14:88:09\novn.kubernetes.io/network_types: geneve\novn.kubernetes.io/routed: \"true\"\n...\npodIP: 10.16.0.9\npodIPs:\n- ip: 10.16.0.9\n- ip: fd00:10:16::9\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/eip-snat/","title":"EIP \u548c SNAT \u914d\u7f6e","text":"<p>\u8be5\u914d\u7f6e\u9488\u5bf9\u9ed8\u8ba4 VPC \u4e0b\u7684\u7f51\u7edc\uff0c\u7528\u6237\u81ea\u5b9a\u4e49 VPC \u8bf7\u53c2\u8003 VPC \u7f51\u5173</p> <p>Kube-OVN \u652f\u6301\u5229\u7528 OVN \u4e2d\u7684 L3 Gateway \u529f\u80fd\u6765\u5b9e\u73b0 Pod \u7ea7\u522b\u7684 SNAT \u548c EIP \u529f\u80fd\u3002 \u901a\u8fc7\u4f7f\u7528 SNAT\uff0c\u4e00\u7ec4 Pod \u53ef\u4ee5\u5171\u4eab\u4e00\u4e2a IP \u5730\u5740\u5bf9\u5916\u8fdb\u884c\u8bbf\u95ee\u3002 \u901a\u8fc7 EIP \u7684\u529f\u80fd\uff0c\u4e00\u4e2a Pod \u53ef\u4ee5\u76f4\u63a5\u548c\u4e00\u4e2a\u5916\u90e8 IP \u5173\u8054\uff0c \u5916\u90e8\u670d\u52a1\u53ef\u4ee5\u901a\u8fc7 EIP \u76f4\u63a5\u8bbf\u95ee Pod\uff0cPod \u4e5f\u5c06\u901a\u8fc7\u8fd9\u4e2a EIP \u8bbf\u95ee\u5916\u90e8\u670d\u52a1\u3002</p> <p></p>"},{"location":"guide/eip-snat/#_1","title":"\u51c6\u5907\u5de5\u4f5c","text":"<ul> <li>\u4e3a\u4e86\u4f7f\u7528 OVN \u7684 L3 Gateway \u80fd\u529b\uff0c\u5fc5\u987b\u5c06\u4e00\u4e2a\u5355\u72ec\u7684\u7f51\u5361\u63a5\u5165 OVS \u7f51\u6865\u4e2d\u8fdb\u884c Overlay \u548c Underlay \u7f51\u7edc\u7684\u6253\u901a\uff0c \u4e3b\u673a\u5fc5\u987b\u6709\u5176\u4ed6\u7684\u7f51\u5361\u7528\u4e8e\u8fd0\u7ef4\u7ba1\u7406\u3002</li> <li>\u7531\u4e8e\u7ecf\u8fc7 NAT \u540e\u7684\u6570\u636e\u5305\u4f1a\u76f4\u63a5\u8fdb\u5165 Underlay \u7f51\u7edc\uff0c\u5fc5\u987b\u786e\u8ba4\u5f53\u524d\u7684\u7f51\u7edc\u67b6\u6784\u4e0b\u6b64\u7c7b\u6570\u636e\u5305\u53ef\u4ee5\u5b89\u5168\u901a\u8fc7\u3002</li> <li>\u76ee\u524d EIP \u548c SNAT \u5730\u5740\u6ca1\u6709\u51b2\u7a81\u68c0\u6d4b\uff0c\u9700\u8981\u7ba1\u7406\u5458\u624b\u52a8\u5206\u914d\u907f\u514d\u5730\u5740\u51b2\u7a81\u3002</li> </ul>"},{"location":"guide/eip-snat/#_2","title":"\u521b\u5efa\u914d\u7f6e\u6587\u4ef6","text":"<p>\u5728 <code>kube-system</code> \u4e0b\u521b\u5efa ConfigMap <code>ovn-external-gw-config</code>\uff1a</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: ovn-external-gw-config\nnamespace: kube-system\ndata:\nenable-external-gw: \"true\"\nexternal-gw-nodes: \"kube-ovn-worker\"\nexternal-gw-nic: \"eth1\"\nexternal-gw-addr: \"172.56.0.1/16\"\nnic-ip: \"172.56.0.254/16\"\nnic-mac: \"16:52:f3:13:6a:25\"\n</code></pre> <ul> <li><code>enable-external-gw</code>: \u662f\u5426\u5f00\u542f SNAT \u548c EIP \u529f\u80fd\u3002</li> <li><code>type</code>: <code>centrailized</code> \u6216 <code>distributed</code>\uff0c \u9ed8\u8ba4\u4e3a <code>centralized</code> \u5982\u679c\u4f7f\u7528 <code>distributed</code>\uff0c\u5219\u96c6\u7fa4\u6240\u6709\u8282\u70b9\u90fd\u9700\u8981\u6709\u540c\u540d\u7f51\u5361\u6765\u627f\u62c5\u7f51\u5173\u529f\u80fd\u3002</li> <li><code>external-gw-nodes</code>: <code>centralized</code> \u6a21\u5f0f\u4e0b\uff0c\u627f\u62c5\u7f51\u5173\u4f5c\u7528\u7684\u8282\u70b9\u540d\uff0c\u9017\u53f7\u5206\u9694\u3002</li> <li><code>external-gw-nic</code>: \u8282\u70b9\u4e0a\u627f\u62c5\u7f51\u5173\u4f5c\u7528\u7684\u7f51\u5361\u540d\u3002</li> <li><code>external-gw-addr</code>: \u7269\u7406\u7f51\u7edc\u7f51\u5173\u7684 IP \u548c\u63a9\u7801\u3002</li> <li><code>nic-ip</code>,<code>nic-mac</code>: \u5206\u914d\u7ed9\u903b\u8f91\u7f51\u5173\u7aef\u53e3\u7684 IP \u548c Mac\uff0c\u9700\u4e3a\u7269\u7406\u6bb5\u672a\u88ab\u5360\u7528\u7684 IP \u548c Mac\u3002</li> </ul>"},{"location":"guide/eip-snat/#ovn-ovs","title":"\u89c2\u5bdf OVN \u548c OVS \u72b6\u6001\u786e\u8ba4\u914d\u7f6e\u751f\u6548","text":"<p>\u68c0\u67e5 OVN-NB \u72b6\u6001, \u786e\u8ba4 <code>ovn-external</code> \u903b\u8f91\u4ea4\u6362\u673a\u5b58\u5728\uff0c\u5e76\u4e14 <code>ovn-cluster-ovn-external</code> \u903b\u8f91\u8def\u7531\u5668\u7aef\u53e3\u4e0a \u7ed1\u5b9a\u4e86\u6b63\u786e\u7684\u5730\u5740\u548c chassis\u3002</p> <pre><code># kubectl ko nbctl show\nswitch 3de4cea7-1a71-43f3-8b62-435a57ef16a6 (ovn-external)\nport ln-ovn-external\n        type: localnet\n        addresses: [\"unknown\"]\nport ovn-external-ovn-cluster\n        type: router\n        router-port: ovn-cluster-ovn-external\nrouter e1eb83ad-34be-4ed5-9a02-fcc8b1d357c4 (ovn-cluster)\nport ovn-cluster-ovn-external\n        mac: \"ac:1f:6b:2d:33:f1\"\nnetworks: [\"172.56.0.100/16\"]\ngateway chassis: [a5682814-2e2c-46dd-9c1c-6803ef0dab66]\n</code></pre> <p>\u68c0\u67e5 OVS \u72b6\u6001\uff0c\u786e\u8ba4\u76f8\u5e94\u7684\u7f51\u5361\u5df2\u7ecf\u6865\u63a5\u8fdb <code>br-external</code> \u7f51\u6865\uff1a</p> <pre><code># kubectl ko vsctl ${gateway node name} show\ne7d81150-7743-4d6e-9e6f-5c688232e130\n    Bridge br-external\n        Port br-external\n            Interface br-external\n                type: internal\n        Port eno2\n            Interface eno2\n        Port patch-ln-ovn-external-to-br-int\n            Interface patch-ln-ovn-external-to-br-int\n                type: patch\n                options: {peer=patch-br-int-to-ln-ovn-external}\n</code></pre>"},{"location":"guide/eip-snat/#pod-eip-snat","title":"Pod \u914d\u7f6e EIP \u548c SNAT","text":"<p>\u53ef\u901a\u8fc7\u5728 Pod \u4e0a\u589e\u52a0 <code>ovn.kubernetes.io/snat</code> \u6216 <code>ovn.kubernetes.io/eip</code> annotation \u6765\u5206\u522b\u914d\u7f6e SNAT \u548c EIP\uff1a</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: pod-gw\nannotations:\novn.kubernetes.io/snat: 172.56.0.200\nspec:\ncontainers:\n- name: snat-pod\nimage: docker.io/library/nginx:alpine\n---\napiVersion: v1\nkind: Pod\nmetadata:\nname: pod-gw\nannotations:\novn.kubernetes.io/eip: 172.56.0.233\nspec:\ncontainers:\n- name: eip-pod\nimage: docker.io/library/nginx:alpine\n</code></pre> <p>\u53ef\u901a\u8fc7 kubectl \u6216\u5176\u4ed6\u5de5\u5177\u52a8\u6001\u8c03\u6574 Pod \u6240\u914d\u7f6e\u7684 EIP \u6216 SNAT \u89c4\u5219\uff0c\u66f4\u6539\u65f6\u8bf7\u6ce8\u610f\u8981\u540c\u65f6\u5220\u9664 <code>ovn.kubernetes.io/routed</code> annotation \u89e6\u53d1\u8def\u7531\u7684\u53d8\u66f4\uff1a</p> <pre><code>kubectl annotate pod pod-gw ovn.kubernetes.io/eip=172.56.0.221 --overwrite\nkubectl annotate pod pod-gw ovn.kubernetes.io/routed-\n</code></pre> <p>\u5f53 EIP \u6216 SNAT \u89c4\u5219\u751f\u6548\u540e\uff0c<code>ovn.kubernetes.io/routed</code> annotation \u4f1a\u88ab\u91cd\u65b0\u6dfb\u52a0\u3002</p>"},{"location":"guide/eip-snat/#_3","title":"\u9ad8\u7ea7\u914d\u7f6e","text":"<p><code>kube-ovn-controller</code> \u7684\u90e8\u5206\u542f\u52a8\u53c2\u6570\u53ef\u5bf9 SNAT \u548c EIP \u529f\u80fd\u8fdb\u884c\u9ad8\u9636\u914d\u7f6e\uff1a</p> <ul> <li><code>--external-gateway-config-ns</code>: Configmap <code>ovn-external-gw-config</code> \u6240\u5c5e Namespace\uff0c \u9ed8\u8ba4\u4e3a <code>kube-system</code>\u3002</li> <li><code>--external-gateway-net</code>: \u7269\u7406\u7f51\u5361\u6240\u6865\u63a5\u7684\u7f51\u6865\u540d\uff0c\u9ed8\u8ba4\u4e3a <code>external</code>\u3002</li> <li><code>--external-gateway-vlanid</code>: \u7269\u7406\u7f51\u7edc Vlan Tag \u53f7\uff0c\u9ed8\u8ba4\u4e3a 0\uff0c \u5373\u4e0d\u4f7f\u7528 Vlan\u3002</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/ippool/","title":"IP \u6c60\u4f7f\u7528","text":"<p>IP \u6c60\uff08IPPool\uff09\u662f\u6bd4\u5b50\u7f51\uff08Subnet\uff09\u66f4\u7ec6\u529b\u5ea6\u7684 IPAM \u7ba1\u7406\u5355\u5143\u3002\u4f60\u53ef\u4ee5\u901a\u8fc7 IP \u6c60\u5c06\u5b50\u7f51\u7f51\u6bb5\u7ec6\u5206\u4e3a\u591a\u4e2a\u5355\u5143\uff0c\u6bcf\u4e2a\u5355\u5143\u7ed1\u5b9a\u4e00\u4e2a\u6216\u591a\u4e2a\u547d\u540d\u7a7a\u95f4\uff08Namespace\uff09\u3002</p>"},{"location":"guide/ippool/#_1","title":"\u4f7f\u7528\u65b9\u6cd5","text":"<p>\u4f7f\u7528\u793a\u4f8b\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: IPPool\nmetadata:\nname: pool-1\nspec:\nsubnet: ovn-default\nips:\n- \"10.16.0.201\"\n- \"10.16.0.210/30\"\n- \"10.16.0.220..10.16.0.230\"\nnamespaces:\n- ns-1\n</code></pre> <p>\u5b57\u6bb5\u8bf4\u660e\uff1a</p> \u540d\u79f0 \u7528\u9014 \u5907\u6ce8 subnet \u6307\u5b9a\u6240\u5c5e\u5b50\u7f51 \u5fc5\u586b ips \u6307\u5b9a\u5305\u542b\u7684 IP \u8303\u56f4 \u652f\u6301 \u3001 \u4ee5\u53ca .. \u4e09\u79cd\u683c\u5f0f\uff0c\u652f\u6301 IPv6\u3002 namespaces \u7ed1\u5b9a\u547d\u540d\u7a7a\u95f4 \u53ef\u9009"},{"location":"guide/ippool/#_2","title":"\u6ce8\u610f\u4e8b\u9879","text":"<ol> <li>\u4e3a\u4fdd\u8bc1\u4e0e Workload \u901a\u7528 IP Pool \u56fa\u5b9a\u5730\u5740 \u7684\u517c\u5bb9\u6027\uff0cIP \u6c60\u7684\u540d\u79f0\u4e0d\u80fd\u662f\u4e00\u4e2a IP \u5730\u5740\uff1b</li> <li>IP \u6c60\u7684 <code>.spec.ips</code> \u53ef\u6307\u5b9a\u8d85\u51fa\u5b50\u7f51\u8303\u56f4\u7684 IP \u5730\u5740\uff0c\u4f46\u5b9e\u9645\u6709\u6548\u7684 IP \u5730\u5740\u662f <code>.spec.ips</code> \u4e0e\u5b50\u7f51 CIDR \u7684\u4ea4\u96c6\uff1b</li> <li>\u540c\u4e00\u4e2a\u5b50\u7f51\u7684\u4e0d\u540c IP \u6c60\uff0c\u4e0d\u80fd\u5305\u542b\u76f8\u540c\u7684\uff08\u6709\u6548\uff09IP \u5730\u5740\uff1b</li> <li>IP \u6c60\u7684 <code>.spec.ips</code> \u53ef\u52a8\u6001\u4fee\u6539\uff1b</li> <li>IP \u6c60\u4f1a\u7ee7\u627f\u5b50\u7f51\u7684\u4fdd\u7559 IP\uff0c\u4ece IP \u6c60\u968f\u673a\u5206\u914d IP \u5730\u5740\u65f6\uff0c\u4f1a\u8df3\u8fc7\u5305\u542b\u5728 IP \u6c60\u4e2d\u7684\u4fdd\u7559 IP\uff1b</li> <li>\u4ece\u5b50\u7f51\u968f\u673a\u5206\u914d IP \u5730\u5740\u65f6\uff0c\u53ea\u4f1a\u4ece\u5b50\u7f51\u6240\u6709 IP \u6c60\u4ee5\u5916\u7684\u8303\u56f4\u5206\u914d\u3002</li> </ol> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/loadbalancer-service/","title":"LoadBalancer \u7c7b\u578b Service","text":"<p>Kube-OVN \u5df2\u7ecf\u652f\u6301\u4e86 VPC \u548c VPC \u7f51\u5173\u7684\u5b9e\u73b0\uff0c\u5177\u4f53\u914d\u7f6e\u53ef\u4ee5\u53c2\u8003 VPC \u914d\u7f6e\u3002</p> <p>\u7531\u4e8e VPC \u7f51\u5173\u7684\u4f7f\u7528\u6bd4\u8f83\u590d\u6742\uff0c\u57fa\u4e8e VPC \u7f51\u5173\u7684\u5b9e\u73b0\u505a\u4e86\u7b80\u5316\uff0c\u652f\u6301\u5728\u9ed8\u8ba4 VPC \u4e0b\u521b\u5efa <code>LoadBalancer</code> \u7c7b\u578b\u7684 Service\uff0c\u5b9e\u73b0\u901a\u8fc7 LoadBalancerIP \u6765\u8bbf\u95ee\u9ed8\u8ba4 VPC \u4e0b\u7684 Service\u3002</p> <p>\u9996\u5148\u786e\u8ba4\u73af\u5883\u4e0a\u6ee1\u8db3\u4ee5\u4e0b\u6761\u4ef6\uff1a</p> <ol> <li>\u5b89\u88c5\u4e86 <code>multus-cni</code> \u548c <code>macvlan cni</code>\u3002</li> <li>LoadBalancer Service \u7684\u652f\u6301\uff0c\u662f\u5bf9 VPC \u7f51\u5173\u4ee3\u7801\u8fdb\u884c\u7b80\u5316\u5b9e\u73b0\u7684\uff0c\u4ecd\u7136\u4f7f\u7528 <code>vpc-nat-gw</code> \u7684\u955c\u50cf\uff0c\u4f9d\u8d56 macvlan \u63d0\u4f9b\u591a\u7f51\u5361\u529f\u80fd\u652f\u6301\u3002</li> <li>\u76ee\u524d\u53ea\u652f\u6301\u5728<code>\u9ed8\u8ba4 VPC</code> \u914d\u7f6e\uff0c\u81ea\u5b9a\u4e49 VPC \u4e0b\u7684 LoadBalancer \u652f\u6301\u53ef\u4ee5\u53c2\u8003 VPC \u7684\u6587\u6863 VPC \u914d\u7f6e\u3002</li> </ol>"},{"location":"guide/loadbalancer-service/#vpc-loadbalancer-service","title":"\u9ed8\u8ba4 VPC LoadBalancer Service \u914d\u7f6e\u6b65\u9aa4","text":""},{"location":"guide/loadbalancer-service/#_1","title":"\u5f00\u542f\u7279\u6027\u5f00\u5173","text":"<p>\u4fee\u6539 kube-system namespace \u4e0b\u7684 deployment <code>kube-ovn-controller</code>\uff0c\u5728 <code>args</code> \u4e2d\u589e\u52a0\u53c2\u6570  <code>--enable-lb-svc=true</code>\uff0c\u5f00\u542f\u529f\u80fd\u5f00\u5173\uff0c\u8be5\u53c2\u6570\u9ed8\u8ba4\u4e3a false\u3002</p> <pre><code>containers:\n- args:\n- /kube-ovn/start-controller.sh\n- --default-cidr=10.16.0.0/16\n- --default-gateway=10.16.0.1\n- --default-gateway-check=true\n- --enable-lb-svc=true                  // \u53c2\u6570\u8bbe\u7f6e\u4e3a true\n</code></pre>"},{"location":"guide/loadbalancer-service/#networkattachmentdefinition-crd","title":"\u521b\u5efa NetworkAttachmentDefinition CRD \u8d44\u6e90","text":"<p>\u53c2\u8003\u4ee5\u4e0b yaml\uff0c\u521b\u5efa <code>net-attach-def</code> \u8d44\u6e90:</p> <pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: lb-svc-attachment\nnamespace: kube-system\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.0\",\n\"type\": \"macvlan\",\n\"master\": \"eth0\",                         //\u7269\u7406\u7f51\u5361\uff0c\u6839\u636e\u5b9e\u9645\u60c5\u51b5\u914d\u7f6e\n\"mode\": \"bridge\"\n}'\n</code></pre> <p>\u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0c\u901a\u8fc7\u7269\u7406\u7f51\u5361 <code>eth0</code> \u6765\u5b9e\u73b0\u591a\u7f51\u5361\u529f\u80fd\uff0c\u5982\u679c\u9700\u8981\u4f7f\u7528\u5176\u4ed6\u7269\u7406\u7f51\u5361\uff0c\u53ef\u4ee5\u901a\u8fc7\u4fee\u6539 <code>master</code> \u53d6\u503c\uff0c\u6307\u5b9a\u4f7f\u7528\u7684\u7269\u7406\u7f51\u5361\u540d\u79f0\u3002</p>"},{"location":"guide/loadbalancer-service/#subnet","title":"\u521b\u5efa Subnet","text":"<p>\u521b\u5efa\u7684 Subnet\uff0c\u7528\u4e8e\u7ed9 LoadBalancer Service \u5206\u914d LoadBalancerIP\uff0c\u8be5\u5730\u5740\u6b63\u5e38\u60c5\u51b5\u4e0b\u5728\u96c6\u7fa4\u5916\u5e94\u8be5\u53ef\u4ee5\u8bbf\u95ee\u5230\u3002\u53ef\u4ee5\u914d\u7f6e Underlay Subnet \u7528\u4e8e\u5730\u5740\u5206\u914d\u3002</p> <p>\u53c2\u8003\u4ee5\u4e0b yaml\uff0c\u521b\u5efa\u65b0\u5b50\u7f51\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: attach-subnet\nspec:\nprotocol: IPv4\nprovider: lb-svc-attachment.kube-system    # provider \u683c\u5f0f\u56fa\u5b9a\uff0c\u7531\u4e0a\u4e00\u6b65\u521b\u5efa\u7684 net-attach-def \u8d44\u6e90\u7684 Name.Namespace \u7ec4\u6210\ncidrBlock: 172.18.0.0/16\ngateway: 172.18.0.1\nexcludeIps:\n- 172.18.0.0..172.18.0.10\n</code></pre> <p>Subnet \u4e2d <code>provider</code> \u53c2\u6570\u4ee5 <code>ovn</code> \u6216\u8005\u4ee5 <code>.ovn</code> \u4e3a\u540e\u7f00\u7ed3\u675f\uff0c\u8868\u793a\u8be5\u5b50\u7f51\u662f\u7531 Kube-OVN \u7ba1\u7406\u4f7f\u7528\uff0c\u9700\u8981\u5bf9\u5e94\u521b\u5efa <code>logical switch</code> \u8bb0\u5f55\u3002</p> <p><code>provider</code> \u975e <code>ovn</code> \u6216\u8005\u975e <code>.ovn</code> \u4e3a\u540e\u7f00\u7ed3\u675f\uff0c\u5219 Kube-OVN \u53ea\u63d0\u4f9b IPAM \u529f\u80fd\uff0c\u8bb0\u5f55 IP \u5730\u5740\u5206\u914d\u60c5\u51b5\uff0c\u4e0d\u5bf9\u5b50\u7f51\u505a\u4e1a\u52a1\u903b\u8f91\u5904\u7406\u3002</p>"},{"location":"guide/loadbalancer-service/#loadbalancer-service_1","title":"\u521b\u5efa LoadBalancer Service","text":"<p>\u53c2\u8003\u4ee5\u4e0b yaml\uff0c\u521b\u5efa LoadBalancer Service\uff1a</p> <pre><code>apiVersion: v1\nkind: Service\nmetadata:\nannotations:\nlb-svc-attachment.kube-system.kubernetes.io/logical_switch: attach-subnet   #\u53ef\u9009\novn.kubernetes.io/attachmentprovider: lb-svc-attachment.kube-system          #\u5fc5\u987b\nlabels:\napp: dynamic\nname: test-service\nnamespace: default\nspec:\nloadBalancerIP: 172.18.0.18                                                   #\u53ef\u9009\nports:\n- name: test\nprotocol: TCP\nport: 80\ntargetPort: 80\nselector:\napp: dynamic\nsessionAffinity: None\ntype: LoadBalancer\n</code></pre> <p>\u5728 yaml \u4e2d\uff0cannotation <code>ovn.kubernetes.io/attachmentprovider</code>  \u4e3a\u5fc5\u586b\u9879\uff0c\u53d6\u503c\u7531\u7b2c\u4e00\u6b65\u521b\u5efa\u7684 <code>net-attach-def</code> \u8d44\u6e90\u7684 Name.Namespace \u7ec4\u6210\u3002\u8be5 annotation \u7528\u4e8e\u5728\u521b\u5efa Pod \u65f6\uff0c\u67e5\u627e <code>net-attach-def</code> \u8d44\u6e90\u3002</p> <p>\u53ef\u4ee5\u901a\u8fc7 annotation \u6307\u5b9a\u591a\u7f51\u5361\u5730\u5740\u5206\u914d\u4f7f\u7528\u7684\u5b50\u7f51\u3002annotation key \u683c\u5f0f\u4e3a net-attach-def \u8d44\u6e90\u7684 <code>Name.Namespace.kubernetes.io/logical_switch</code>\u3002\u8be5\u914d\u7f6e\u4e3a<code>\u53ef\u9009</code>\u9009\u9879\uff0c\u5728\u6ca1\u6709\u6307\u5b9a LoadBalancerIP \u5730\u5740\u7684\u60c5\u51b5\u4e0b\uff0c\u5c06\u4ece\u8be5\u5b50\u7f51\u52a8\u6001\u5206\u914d\u5730\u5740\uff0c\u586b\u5145\u5230 LoadBalancerIP \u5b57\u6bb5\u3002</p> <p>\u5982\u679c\u9700\u8981\u9759\u6001\u914d\u7f6e LoadBalancerIP \u5730\u5740\uff0c\u53ef\u4ee5\u914d\u7f6e <code>spec.loadBalancerIP</code> \u5b57\u6bb5\uff0c\u8be5\u5730\u5740\u9700\u8981\u5728\u6307\u5b9a\u5b50\u7f51\u7684\u5730\u5740\u8303\u56f4\u5185\u3002</p> <p>\u5728\u6267\u884c yaml \u521b\u5efa Service \u540e\uff0c\u5728 Service \u540c Namespace \u4e0b\uff0c\u53ef\u4ee5\u770b\u5230 Pod \u542f\u52a8\u4fe1\u606f\uff1a</p> <pre><code># kubectl get pod\nNAME                                      READY   STATUS    RESTARTS   AGE\nlb-svc-test-service-6869d98dd8-cjvll      1/1     Running   0          107m\n# kubectl get svc\nNAME              TYPE           CLUSTER-IP       EXTERNAL-IP   PORT(S)        AGE\ntest-service      LoadBalancer   10.109.201.193   172.18.0.18   80:30056/TCP   107m\n</code></pre> <p>\u6307\u5b9a <code>service.spec.loadBalancerIP</code> \u53c2\u6570\u65f6\uff0c\u6700\u7ec8\u5c06\u8be5\u53c2\u6570\u8d4b\u503c\u7ed9 service external-ip \u5b57\u6bb5\u3002\u4e0d\u6307\u5b9a\u7684\u60c5\u51b5\u4e0b\uff0c\u8be5\u53c2\u6570\u4e3a\u968f\u673a\u5206\u914d\u503c\u3002</p> <p>\u67e5\u770b\u6d4b\u8bd5 Pod \u7684 yaml \u8f93\u51fa\uff0c\u5b58\u5728\u591a\u7f51\u5361\u5206\u914d\u7684\u5730\u5740\u4fe1\u606f\uff1a</p> <pre><code># kubectl get pod -o yaml lb-svc-test-service-6869d98dd8-cjvll\napiVersion: v1\nkind: Pod\nmetadata:\n  annotations:\n    k8s.v1.cni.cncf.io/network-status: |-\n      [{\n\"name\": \"kube-ovn\",\n          \"ips\": [\n\"10.16.0.2\"\n],\n          \"default\": true,\n          \"dns\": {}\n},{\n\"name\": \"default/test-service\",\n          \"interface\": \"net1\",\n          \"mac\": \"ba:85:f7:02:9f:42\",\n          \"dns\": {}\n}]\nk8s.v1.cni.cncf.io/networks: default/test-service\n    k8s.v1.cni.cncf.io/networks-status: |-\n      [{\n\"name\": \"kube-ovn\",\n          \"ips\": [\n\"10.16.0.2\"\n],\n          \"default\": true,\n          \"dns\": {}\n},{\n\"name\": \"default/test-service\",\n          \"interface\": \"net1\",\n          \"mac\": \"ba:85:f7:02:9f:42\",\n          \"dns\": {}\n}]\novn.kubernetes.io/allocated: \"true\"\novn.kubernetes.io/cidr: 10.16.0.0/16\n    ovn.kubernetes.io/gateway: 10.16.0.1\n    ovn.kubernetes.io/ip_address: 10.16.0.2\n    ovn.kubernetes.io/logical_router: ovn-cluster\n    ovn.kubernetes.io/logical_switch: ovn-default\n    ovn.kubernetes.io/mac_address: 00:00:00:45:F4:29\n    ovn.kubernetes.io/pod_nic_type: veth-pair\n    ovn.kubernetes.io/routed: \"true\"\ntest-service.default.kubernetes.io/allocated: \"true\"\ntest-service.default.kubernetes.io/cidr: 172.18.0.0/16\n    test-service.default.kubernetes.io/gateway: 172.18.0.1\n    test-service.default.kubernetes.io/ip_address: 172.18.0.18\n    test-service.default.kubernetes.io/logical_switch: attach-subnet\n    test-service.default.kubernetes.io/mac_address: 00:00:00:AF:AA:BF\n    test-service.default.kubernetes.io/pod_nic_type: veth-pair\n</code></pre> <p>\u67e5\u770b Service \u7684\u4fe1\u606f\uff1a</p> <pre><code># kubectl get svc -o yaml test-service\napiVersion: v1\nkind: Service\nmetadata:\n  annotations:\n    kubectl.kubernetes.io/last-applied-configuration: |\n{\"apiVersion\":\"v1\",\"kind\":\"Service\",\"metadata\":{\"annotations\":{\"test-service.default.kubernetes.io/logical_switch\":\"attach-subnet\"},\"labels\":{\"app\":\"dynamic\"},\"name\":\"test-service\",\"namespace\":\"default\"},\"spec\":{\"ports\":[{\"name\":\"test\",\"port\":80,\"protocol\":\"TCP\",\"targetPort\":80}],\"selector\":{\"app\":\"dynamic\"},\"sessionAffinity\":\"None\",\"type\":\"LoadBalancer\"}}\novn.kubernetes.io/vpc: ovn-cluster\n    test-service.default.kubernetes.io/logical_switch: attach-subnet\n  creationTimestamp: \"2022-06-15T09:01:58Z\"\nlabels:\n    app: dynamic\n  name: test-service\n  namespace: default\n  resourceVersion: \"38485\"\nuid: 161edee1-7f6e-40f5-9e09-5a52c44267d0\nspec:\n  allocateLoadBalancerNodePorts: true\nclusterIP: 10.109.201.193\n  clusterIPs:\n  - 10.109.201.193\n  externalTrafficPolicy: Cluster\n  internalTrafficPolicy: Cluster\n  ipFamilies:\n  - IPv4\n  ipFamilyPolicy: SingleStack\n  ports:\n  - name: test\nnodePort: 30056\nport: 80\nprotocol: TCP\n    targetPort: 80\nselector:\n    app: dynamic\n  sessionAffinity: None\n  type: LoadBalancer\nstatus:\n  loadBalancer:\n    ingress:\n    - ip: 172.18.0.18\n</code></pre>"},{"location":"guide/loadbalancer-service/#loadbalancerip","title":"\u6d4b\u8bd5 LoadBalancerIP \u8bbf\u95ee","text":"<p>\u53c2\u8003\u4ee5\u4e0b yaml, \u521b\u5efa\u6d4b\u8bd5 Pod\uff0c\u4f5c\u4e3a Service \u7684 Endpoints \u63d0\u4f9b\u670d\u52a1:</p> <pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nlabels:\napp: dynamic\nname: dynamic\nnamespace: default\nspec:\nreplicas: 2\nselector:\nmatchLabels:\napp: dynamic\nstrategy:\nrollingUpdate:\nmaxSurge: 25%\nmaxUnavailable: 25%\ntype: RollingUpdate\ntemplate:\nmetadata:\ncreationTimestamp: null\nlabels:\napp: dynamic\nspec:\ncontainers:\n- image: docker.io/library/nginx:alpine\nimagePullPolicy: IfNotPresent\nname: nginx\ndnsPolicy: ClusterFirst\nrestartPolicy: Always\n</code></pre> <p>\u6b63\u5e38\u60c5\u51b5\u4e0b\uff0c\u63d0\u4f9b\u7684\u5b50\u7f51\u5730\u5740\uff0c\u5728\u96c6\u7fa4\u5916\u5e94\u8be5\u53ef\u4ee5\u8bbf\u95ee\u5230\u3002\u4e3a\u4e86\u7b80\u5355\u9a8c\u8bc1\uff0c\u5728\u96c6\u7fa4\u5185\u8bbf\u95ee Service \u7684 <code>LoadBalancerIP:Port</code>\uff0c\u67e5\u770b\u662f\u5426\u6b63\u5e38\u8bbf\u95ee\u6210\u529f\u3002</p> <pre><code># curl 172.18.0.11:80\n&lt;html&gt;\n&lt;head&gt;\n        &lt;title&gt;Hello World!&lt;/title&gt;\n        &lt;link href='//fonts.googleapis.com/css?family=Open+Sans:400,700' rel='stylesheet' type='text/css'&gt;\n        &lt;style&gt;\n        body {\nbackground-color: white;\ntext-align: center;\npadding: 50px;\nfont-family: \"Open Sans\",\"Helvetica Neue\",Helvetica,Arial,sans-serif;\n}\n#logo {\nmargin-bottom: 40px;\n}\n&lt;/style&gt;\n&lt;/head&gt;\n&lt;body&gt;\n                &lt;h1&gt;Hello World!&lt;/h1&gt;\n                                &lt;h3&gt;Links found&lt;/h3&gt;\n        &lt;h3&gt;I am on  dynamic-7d8d7874f5-hsgc4&lt;/h3&gt;\n        &lt;h3&gt;Cookie                  =&lt;/h3&gt;\n                                        &lt;b&gt;KUBERNETES&lt;/b&gt; listening in 443 available at tcp://10.96.0.1:443&lt;br /&gt;\n                                                &lt;h3&gt;my name is hanhouchao!&lt;/h3&gt;\n                        &lt;h3&gt; RequestURI='/'&lt;/h3&gt;\n&lt;/body&gt;\n&lt;/html&gt;\n</code></pre> <p>\u8fdb\u5165 Service \u521b\u5efa\u7684 Pod\uff0c\u67e5\u770b\u7f51\u7edc\u7684\u4fe1\u606f</p> <pre><code># ip a\n4: net1@if62: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1500 qdisc noqueue state UP group default\n    link/ether ba:85:f7:02:9f:42 brd ff:ff:ff:ff:ff:ff link-netnsid 0\ninet 172.18.0.18/16 scope global net1\n       valid_lft forever preferred_lft forever\n    inet6 fe80::b885:f7ff:fe02:9f42/64 scope link\n       valid_lft forever preferred_lft forever\n36: eth0@if37: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1400 qdisc noqueue state UP group default\n    link/ether 00:00:00:45:f4:29 brd ff:ff:ff:ff:ff:ff link-netnsid 0\ninet 10.16.0.2/16 brd 10.16.255.255 scope global eth0\n       valid_lft forever preferred_lft forever\n    inet6 fe80::200:ff:fe45:f429/64 scope link\n       valid_lft forever preferred_lft forever\n\n# ip rule\n0: from all lookup local\n32764: from all iif eth0 lookup 100\n32765: from all iif net1 lookup 100\n32766: from all lookup main\n32767: from all lookup default\n\n# ip route show table 100\ndefault via 172.18.0.1 dev net1\n10.109.201.193 via 10.16.0.1 dev eth0\n172.18.0.0/16 dev net1 scope link\n\n# iptables -t nat -L -n -v\nChain PREROUTING (policy ACCEPT 0 packets, 0 bytes)\npkts bytes target     prot opt in     out     source               destination\n    0     0 DNAT       tcp  --  *      *       0.0.0.0/0            172.18.0.18          tcp dpt:80 to:10.109.201.193:80\n\nChain INPUT (policy ACCEPT 0 packets, 0 bytes)\npkts bytes target     prot opt in     out     source               destination\n\nChain OUTPUT (policy ACCEPT 0 packets, 0 bytes)\npkts bytes target     prot opt in     out     source               destination\n\nChain POSTROUTING (policy ACCEPT 0 packets, 0 bytes)\npkts bytes target     prot opt in     out     source               destination\n    0     0 MASQUERADE  all  --  *      *       0.0.0.0/0            10.109.201.193\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/mirror/","title":"\u6d41\u91cf\u955c\u50cf","text":"<p>\u6d41\u91cf\u955c\u50cf\u529f\u80fd\u53ef\u4ee5\u5c06\u8fdb\u51fa\u5bb9\u5668\u7f51\u7edc\u7684\u6570\u636e\u5305\u8fdb\u884c\u590d\u5236\u5230\u4e3b\u673a\u7684\u7279\u5b9a\u7f51\u5361\u3002\u7ba1\u7406\u5458\u6216\u5f00\u53d1\u8005 \u53ef\u4ee5\u901a\u8fc7\u76d1\u542c\u8fd9\u5757\u7f51\u5361\u83b7\u5f97\u5b8c\u6574\u7684\u5bb9\u5668\u7f51\u7edc\u6d41\u91cf\u6765\u8fdb\u4e00\u6b65\u8fdb\u884c\u5206\u6790\uff0c\u76d1\u63a7\uff0c\u5b89\u5168\u5ba1\u8ba1\u7b49\u64cd\u4f5c\u3002 \u4e5f\u53ef\u548c\u4f20\u7edf\u7684 NPM \u5bf9\u63a5\u83b7\u53d6\u66f4\u7ec6\u7c92\u5ea6\u7684\u6d41\u91cf\u76d1\u63a7\u3002</p> <p>\u6d41\u91cf\u955c\u50cf\u529f\u80fd\u4f1a\u5e26\u6765\u4e00\u5b9a\u7684\u6027\u80fd\u635f\u5931\uff0c\u6839\u636e CPU \u6027\u80fd\u4ee5\u53ca\u6d41\u91cf\u7684\u7279\u5f81\uff0c\u4f1a\u6709 5%~10% \u7684 \u989d\u5916 CPU \u6d88\u8017\u3002</p> <p></p>"},{"location":"guide/mirror/#_2","title":"\u5168\u5c40\u6d41\u91cf\u955c\u50cf\u914d\u7f6e","text":"<p>\u6d41\u91cf\u955c\u50cf\u529f\u80fd\u9ed8\u8ba4\u4e3a\u5173\u95ed\u72b6\u6001\uff0c\u5982\u679c\u9700\u8981\u5f00\u542f\u8bf7\u4fee\u6539 <code>kube-ovn-cni</code> DaemonSet \u7684\u542f\u52a8\u53c2\u6570\uff1a</p> <ul> <li><code>--enable-mirror=true</code>\uff1a \u662f\u5426\u5f00\u542f\u6d41\u91cf\u955c\u50cf\u3002</li> <li><code>--mirror-iface=mirror0</code>: \u6d41\u91cf\u955c\u50cf\u6240\u590d\u5236\u5230\u7684\u7f51\u5361\u540d\u3002\u8be5\u7f51\u5361\u53ef\u4e3a\u4e3b\u673a\u4e0a\u5df2\u5b58\u5728\u7684\u4e00\u5757\u7269\u7406\u7f51\u5361\uff0c \u6b64\u65f6\u8be5\u7f51\u5361\u4f1a\u88ab\u6865\u63a5\u8fdb br-int \u7f51\u6865\uff0c\u955c\u50cf\u6d41\u91cf\u4f1a\u76f4\u63a5\u63a5\u5165\u5e95\u5c42\u4ea4\u6362\u673a\u3002\u82e5\u7f51\u5361\u540d\u4e0d\u5b58\u5728\uff0cKube-OVN \u4f1a\u81ea\u52a8 \u521b\u5efa\u4e00\u5757\u540c\u540d\u7684\u865a\u62df\u7f51\u5361\uff0c\u7ba1\u7406\u5458\u6216\u5f00\u53d1\u8005\u53ef\u4ee5\u5728\u5bbf\u4e3b\u673a\u4e0a\u901a\u8fc7\u8be5\u7f51\u5361\u83b7\u53d6\u5f53\u524d\u8282\u70b9\u6240\u6709\u6d41\u91cf\u3002\u9ed8\u8ba4\u4e3a <code>mirror0</code>\u3002</li> </ul> <p>\u63a5\u4e0b\u6765\u53ef\u4ee5\u7528 tcpdump \u6216\u5176\u4ed6\u6d41\u91cf\u5206\u6790\u5de5\u5177\u76d1\u542c <code>mirror0</code> \u4e0a\u7684\u6d41\u91cf\uff1a</p> <pre><code>tcpdump -ni mirror0\n</code></pre>"},{"location":"guide/mirror/#pod","title":"Pod \u7ea7\u522b\u6d41\u91cf\u955c\u50cf\u914d\u7f6e","text":"<p>\u5982\u679c\u53ea\u9700\u5bf9\u90e8\u5206 Pod \u6d41\u91cf\u8fdb\u884c\u955c\u50cf\uff0c\u5219\u9700\u8981\u5173\u95ed\u5168\u5c40\u7684\u6d41\u91cf\u955c\u50cf\u529f\u80fd\uff0c\u7136\u540e\u5728\u7279\u5b9a Pod \u4e0a\u589e\u52a0 <code>ovn.kubernetes.io/mirror</code> annotation \u6765\u5f00\u542f Pod \u7ea7\u522b\u6d41\u91cf\u955c\u50cf\u3002</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: mirror-pod\nnamespace: ls1\nannotations:\novn.kubernetes.io/mirror: \"true\"\nspec:\ncontainers:\n- name: mirror-pod\nimage: docker.io/library/nginx:alpine\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/networkpolicy-log/","title":"NetworkPolicy \u65e5\u5fd7","text":"<p>NetworkPolicy \u4e3a Kubernetes \u63d0\u4f9b\u7684\u7f51\u7edc\u7b56\u7565\u63a5\u53e3\uff0cKube-OVN \u901a\u8fc7 OVN \u7684 ACL \u8fdb\u884c\u4e86\u5b9e\u73b0\u3002 \u4f7f\u7528\u4e86 NetworkPolicy \u540e\u5982\u679c\u51fa\u73b0\u7f51\u7edc\u4e0d\u901a\u7684\u60c5\u51b5\uff0c\u96be\u4ee5\u5224\u65ad\u662f\u7f51\u7edc\u6545\u969c\u95ee\u9898\u8fd8\u662f NetworkPolicy \u89c4\u5219\u8bbe\u7f6e\u95ee\u9898\u5bfc\u81f4\u7684\u7f51\u7edc\u4e2d\u65ad\u3002 Kube-OVN \u63d0\u4f9b\u4e86 NetworkPolicy \u65e5\u5fd7\u529f\u80fd\uff0c\u5e2e\u52a9\u7ba1\u7406\u5458\u5feb\u901f\u5b9a\u4f4d NetworkPolicy Drop \u89c4\u5219\u662f\u5426\u547d\u4e2d\uff0c\u5e76\u8bb0\u5f55\u6709\u54ea\u4e9b\u975e\u6cd5\u8bbf\u95ee\u3002</p> <p>NetworkPolicy \u65e5\u5fd7\u529f\u80fd\u4e00\u65e6\u5f00\u542f\uff0c\u5bf9\u6bcf\u4e2a\u547d\u4e2d Drop \u89c4\u5219\u7684\u6570\u636e\u5305\u90fd\u9700\u8981\u6253\u5370\u65e5\u5fd7\uff0c\u4f1a\u5e26\u6765\u989d\u5916\u6027\u80fd\u5f00\u9500\u3002 \u5728\u6076\u610f\u653b\u51fb\u4e0b\uff0c\u77ed\u65f6\u95f4\u5927\u91cf\u65e5\u5fd7\u53ef\u80fd\u4f1a\u8017\u5c3d CPU\u3002\u6211\u4eec\u5efa\u8bae\u5728\u751f\u4ea7\u73af\u5883\u9ed8\u8ba4\u5173\u95ed\u65e5\u5fd7\u529f\u80fd\uff0c\u5728\u9700\u8981\u6392\u67e5\u95ee\u9898\u65f6\uff0c\u52a8\u6001\u5f00\u542f\u65e5\u5fd7\u3002</p>"},{"location":"guide/networkpolicy-log/#networkpolicy_1","title":"\u5f00\u542f NetworkPolicy \u65e5\u5fd7","text":"<p>\u5728\u9700\u8981\u5f00\u542f\u65e5\u5fd7\u8bb0\u5f55\u7684 NetworkPolicy \u4e2d\u589e\u52a0 annotation <code>ovn.kubernetes.io/enable_log</code>\uff0c\u5982\u4e0b\u6240\u793a\uff1a</p> <pre><code>apiVersion: networking.k8s.io/v1\nkind: NetworkPolicy\nmetadata:\nname: default-deny-ingress\nnamespace: kube-system\nannotations:\novn.kubernetes.io/enable_log: \"true\"\nspec:\npodSelector: {}\npolicyTypes:\n- Ingress\n</code></pre> <p>\u63a5\u4e0b\u6765\u53ef\u4ee5\u5728\u5bf9\u5e94 Pod \u6240\u5728\u4e3b\u673a\u7684 <code>/var/log/ovn/ovn-controller.log</code> \u4e2d\u89c2\u5bdf\u5230\u88ab\u4e22\u5f03\u6570\u636e\u5305\u7684\u65e5\u5fd7\uff1a</p> <pre><code># tail -f /var/log/ovn/ovn-controller.log\n2022-07-20T05:55:03.229Z|00394|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: udp,vlan_tci=0x0000,dl_src=00:00:00:21:b7:d1,dl_dst=00:00:00:8d:0b:86,nw_src=10.16.0.10,nw_dst=10.16.0.7,nw_tos=0,nw_ecn=0,nw_ttl=63,tp_src=54343,tp_dst=53\n2022-07-20T05:55:06.229Z|00395|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: udp,vlan_tci=0x0000,dl_src=00:00:00:21:b7:d1,dl_dst=00:00:00:8d:0b:86,nw_src=10.16.0.9,nw_dst=10.16.0.7,nw_tos=0,nw_ecn=0,nw_ttl=63,tp_src=44187,tp_dst=53\n2022-07-20T05:55:08.230Z|00396|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: udp,vlan_tci=0x0000,dl_src=00:00:00:21:b7:d1,dl_dst=00:00:00:8d:0b:86,nw_src=10.16.0.10,nw_dst=10.16.0.7,nw_tos=0,nw_ecn=0,nw_ttl=63,tp_src=54274,tp_dst=53\n2022-07-20T05:55:11.231Z|00397|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: udp,vlan_tci=0x0000,dl_src=00:00:00:21:b7:d1,dl_dst=00:00:00:8d:0b:86,nw_src=10.16.0.9,nw_dst=10.16.0.7,nw_tos=0,nw_ecn=0,nw_ttl=63,tp_src=32778,tp_dst=53\n2022-07-20T05:55:11.231Z|00398|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: udp,vlan_tci=0x0000,dl_src=00:00:00:21:b7:d1,dl_dst=00:00:00:8d:0b:86,nw_src=10.16.0.9,nw_dst=10.16.0.7,nw_tos=0,nw_ecn=0,nw_ttl=63,tp_src=34188,tp_dst=53\n2022-07-20T05:55:13.231Z|00399|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: udp,vlan_tci=0x0000,dl_src=00:00:00:21:b7:d1,dl_dst=00:00:00:8d:0b:86,nw_src=10.16.0.10,nw_dst=10.16.0.7,nw_tos=0,nw_ecn=0,nw_ttl=63,tp_src=43290,tp_dst=53\n2022-07-20T05:55:22.096Z|00400|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: icmp,vlan_tci=0x0000,dl_src=00:00:00:6c:42:91,dl_dst=00:00:00:a5:d7:63,nw_src=10.16.0.9,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0\n2022-07-20T05:55:22.097Z|00401|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: icmp,vlan_tci=0x0000,dl_src=00:00:00:6c:42:91,dl_dst=00:00:00:a5:d7:63,nw_src=10.16.0.9,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0\n2022-07-20T05:55:22.098Z|00402|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: icmp,vlan_tci=0x0000,dl_src=00:00:00:6c:42:91,dl_dst=00:00:00:a5:d7:63,nw_src=10.16.0.9,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0\n</code></pre>"},{"location":"guide/networkpolicy-log/#networkpolicy_2","title":"\u5173\u95ed NetworkPolicy \u65e5\u5fd7","text":"<p>\u5c06\u5bf9\u5e94 NetworkPolicy \u4e2d\u7684 annotation <code>ovn.kubernetes.io/enable_log</code> \u8bbe\u7f6e\u4e3a <code>false</code> \u5373\u53ef\u5173\u95ed NetworkPolicy \u65e5\u5fd7\uff1a</p> <pre><code>kubectl annotate networkpolicy -n kube-system default-deny-ingress ovn.kubernetes.io/enable_log=false --overwrite\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/prometheus-grafana/","title":"\u914d\u7f6e\u76d1\u63a7\u548c\u9762\u677f","text":"<p>Kube-OVN \u53ef\u4ee5\u5c06\u7f51\u7edc\u63a7\u5236\u5e73\u9762\u4fe1\u606f\u4ee5\u53ca\u7f51\u7edc\u6570\u636e\u5e73\u9762\u8d28\u91cf\u4fe1\u606f\u6307\u6807\u4ee5 Prometheus \u6240\u652f\u6301\u7684\u683c\u5f0f\u5bf9\u5916\u8f93\u51fa\u3002</p> <p>\u6211\u4eec\u4f7f\u7528 kube-prometheus \u6240\u63d0\u4f9b\u7684 CRD \u6765\u5b9a\u4e49\u76f8\u5e94\u7684 Prometheus \u76d1\u63a7\u89c4\u5219\u3002 \u7528\u6237\u9700\u8981\u9884\u5148\u5b89\u88c5 kube-prometheus \u6765\u542f\u7528\u76f8\u5173\u7684 CRD\u3002Kube-OVN \u6240\u652f\u6301\u7684\u5168\u90e8\u76d1\u63a7\u6307\u6807\u8bf7\u53c2\u8003 Kube-OVN \u76d1\u63a7\u6307\u6807\u3002</p> <p>\u5982\u679c\u4f7f\u7528\u539f\u751f Prometheus \u8bf7\u53c2\u8003\u914d\u7f6e\u539f\u751f Prometheus \u83b7\u53d6\u76d1\u63a7\u6570\u636e \u8fdb\u884c\u914d\u7f6e\u3002</p>"},{"location":"guide/prometheus-grafana/#prometheus-monitor","title":"\u5b89\u88c5 Prometheus Monitor","text":"<p>Kube-OVN \u4f7f\u7528 Prometheus Monitor CRD \u6765\u7ba1\u7406\u76d1\u63a7\u8f93\u51fa\uff1a</p> <pre><code># \u7f51\u54af\u8d28\u91cf\u76f8\u5173\u76d1\u63a7\u6307\u6807\nkubectl apply -f https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/pinger-monitor.yaml\n# kube-ovn-controller \u76f8\u5173\u76d1\u63a7\u6307\u6807\nkubectl apply -f https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/controller-monitor.yaml\n# kube-ovn-cni \u76f8\u5173\u76d1\u63a7\u6307\u6807\nkubectl apply -f https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/cni-monitor.yaml\n# ovn \u76f8\u5173\u76d1\u63a7\u6307\u6807\nkubectl apply -f https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/ovn-monitor.yaml\n</code></pre> <p>Prometheus \u62c9\u53d6\u76d1\u63a7\u65f6\u95f4\u95f4\u9694\u9ed8\u8ba4\u4e3a 15s\uff0c\u5982\u679c\u9700\u8981\u8c03\u6574\u9700\u8981\u4fee\u6539 yaml \u4e2d\u7684 <code>interval</code> \u5b57\u6bb5\u3002</p>"},{"location":"guide/prometheus-grafana/#grafana","title":"\u52a0\u8f7d Grafana \u9762\u677f","text":"<p>Kube-OVN \u8fd8\u63d0\u4f9b\u4e86\u9884\u5148\u5b9a\u4e49\u597d\u7684 Grafana Dashboard \u5c55\u793a\u63a7\u5236\u5e73\u9762\u548c\u6570\u636e\u5e73\u9762\u76f8\u5173\u4fe1\u606f\u3002</p> <p>\u4e0b\u8f7d\u5bf9\u5e94 Dashboard \u6a21\u677f\uff1a</p> <pre><code># \u7f51\u7edc\u8d28\u91cf\u76f8\u5173\u9762\u677f\nwget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/pinger-grafana.json\n# kube-ovn-controller \u76f8\u5173\u9762\u677f\nwget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/controller-grafana.json\n# kube-ovn-cni \u76f8\u5173\u9762\u677f\nwget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/cni-grafana.json\n# ovn \u76f8\u5173\u9762\u677f\nwget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/ovn-grafana.json\n# ovs \u76f8\u5173\u9762\u677f\nwget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/ovs-grafana.json\n</code></pre> <p>\u5728 Grafana \u4e2d\u5bfc\u5165\u6a21\u677f\uff0c\u5e76\u5c06\u6570\u636e\u6e90\u8bbe\u7f6e\u4e3a\u5bf9\u5e94\u7684 Prometheus \u5373\u53ef\u770b\u5230\u5982\u4e0b Dashboard\uff1a</p> <p><code>kube-ovn-controller</code> \u8fd0\u884c\u72b6\u51b5\u76f8\u5173\u9762\u677f\uff1a</p> <p></p> <p><code>kube-ovn-pinger</code> \u7f51\u7edc\u8d28\u91cf\u76f8\u5173\u9762\u677f\uff1a</p> <p></p> <p><code>kube-ovn-cni</code> \u8fd0\u884c\u72b6\u51b5\u76f8\u5173\u9762\u677f\uff1a</p> <p></p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/prometheus/","title":"\u914d\u7f6e\u539f\u751f Prometheus \u83b7\u53d6\u76d1\u63a7\u6570\u636e","text":"<p>Kube-OVN \u63d0\u4f9b\u4e86\u4e30\u5bcc\u7684\u76d1\u63a7\u6570\u636e\uff0c\u7528\u4e8e OVN/OVS \u5065\u5eb7\u72b6\u6001\u68c0\u67e5\uff0c\u4ee5\u53ca\u5bb9\u5668\u7f51\u7edc\u548c\u4e3b\u673a\u7f51\u7edc\u7684\u8fde\u901a\u6027\u68c0\u67e5\u3002Kube-OVN \u914d\u7f6e\u4e86 ServiceMonitor\uff0c\u53ef\u4ee5\u7528\u4e8e Prometheus \u52a8\u6001\u83b7\u53d6\u76d1\u63a7\u6307\u6807\u3002</p> <p>\u5728\u67d0\u4e9b\u60c5\u51b5\u4e0b\uff0c\u53ea\u5b89\u88c5\u4e86 Prometheus Server\uff0c\u6ca1\u6709\u5b89\u88c5\u5176\u4ed6\u7684\u7ec4\u4ef6\uff0c\u53ef\u4ee5\u901a\u8fc7\u4fee\u6539 Prometheus \u7684\u914d\u7f6e\uff0c\u52a8\u6001\u83b7\u53d6\u96c6\u7fa4\u73af\u5883\u7684\u76d1\u63a7\u6570\u636e\u3002</p>"},{"location":"guide/prometheus/#prometheus_1","title":"Prometheus \u914d\u7f6e","text":"<p>\u4ee5\u4e0b\u7684\u914d\u7f6e\u6587\u6863\uff0c\u53c2\u8003\u81ea Prometheus \u670d\u52a1\u53d1\u73b0\u3002</p>"},{"location":"guide/prometheus/#_1","title":"\u6743\u9650\u914d\u7f6e","text":"<p>Prometheus \u90e8\u7f72\u5728\u96c6\u7fa4\u5185\uff0c\u9700\u8981\u901a\u8fc7 k8s apiserver \u6765\u8bbf\u95ee\u96c6\u7fa4\u5185\u7684\u8d44\u6e90\uff0c\u4ece\u800c\u5b9e\u73b0\u67e5\u8be2\u4e1a\u52a1\u7684\u76d1\u63a7\u6570\u636e\u3002</p> <p>\u53c2\u8003\u4ee5\u4e0b yaml\uff0c\u914d\u7f6e Prometheus \u9700\u8981\u7684\u6743\u9650\uff1a</p> <pre><code>apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\nname: prometheus\nrules:\n- apiGroups: [\"\"]\nresources:\n- nodes\n- nodes/proxy\n- services\n- endpoints\n- pods\nverbs: [\"get\", \"list\", \"watch\"]\n- apiGroups:\n- extensions\nresources:\n- ingresses\nverbs: [\"get\", \"list\", \"watch\"]\n- nonResourceURLs: [\"/metrics\"]\nverbs: [\"get\"]\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\nname: prometheus\nnamespace: default\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\nname: prometheus\nroleRef:\napiGroup: rbac.authorization.k8s.io\nkind: ClusterRole\nname: prometheus\nsubjects:\n- kind: ServiceAccount\nname: prometheus\nnamespace: default\n</code></pre>"},{"location":"guide/prometheus/#prometheus_2","title":"Prometheus \u914d\u7f6e\u6587\u4ef6","text":"<p>Prometheus \u7684\u542f\u52a8\uff0c\u4f9d\u8d56\u4e8e\u914d\u7f6e\u6587\u4ef6 prometheus.yml\uff0c\u53ef\u4ee5\u5c06\u8be5\u6587\u4ef6\u5185\u5bb9\u914d\u7f6e\u5728 ConfigMap \u5185\uff0c\u52a8\u6001\u6302\u8f7d\u5230 Pod \u4e2d\u3002</p> <p>\u53c2\u8003\u4ee5\u4e0b yaml\uff0c\u521b\u5efa Prometheus \u4f7f\u7528\u7684 ConfigMap \u6587\u4ef6\uff1a</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: prometheus-config\ndata:\nprometheus.yml: |-\nglobal:\nscrape_interval:     15s \nevaluation_interval: 15s\nscrape_configs:\n- job_name: 'prometheus'\nstatic_configs:\n- targets: ['localhost:9090']\n\n- job_name: 'kubernetes-nodes'\ntls_config:\nca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\nbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\nkubernetes_sd_configs:\n- role: node\n\n- job_name: 'kubernetes-service'\ntls_config:\nca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\nbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\nkubernetes_sd_configs:\n- role: service\n\n- job_name: 'kubernetes-endpoints'\ntls_config:\nca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\nbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\nkubernetes_sd_configs:\n- role: endpoints\n\n- job_name: 'kubernetes-ingress'\ntls_config:\nca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\nbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\nkubernetes_sd_configs:\n- role: ingress\n\n- job_name: 'kubernetes-pods'\ntls_config:\nca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\nbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\nkubernetes_sd_configs:\n- role: pod\n</code></pre> <p>Prometheus \u63d0\u4f9b\u4e86\u57fa\u4e8e\u89d2\u8272\u67e5\u8be2 Kubernetes \u8d44\u6e90\u76d1\u63a7\u7684\u64cd\u4f5c\uff0c\u5177\u4f53\u914d\u7f6e\u53ef\u4ee5\u67e5\u770b\u5b98\u65b9\u6587\u6863 kubernetes_sd_config\u3002</p> <p>\u5728 Kubernetes \u96c6\u7fa4\u4e2d\uff0cPrometheus \u652f\u6301\u67e5\u8be2\u76d1\u63a7\u6307\u6807\u7684\u89d2\u8272\u5305\u542b node\u3001service\u3001pod\u3001endpoints \u548c ingress\u3002\u5728 ConfigMap \u914d\u7f6e\u6587\u4ef6\u4e2d\u7ed9\u51fa\u4e86\u4ee5\u4e0a\u5168\u90e8\u8d44\u6e90\u7684\u76d1\u63a7\u67e5\u8be2\u914d\u7f6e\u793a\u4f8b\uff0c\u53ef\u4ee5\u6839\u636e\u9700\u8981\u9009\u62e9\u914d\u7f6e\u3002</p>"},{"location":"guide/prometheus/#prometheus_3","title":"Prometheus \u90e8\u7f72","text":"<p>\u53c2\u8003\u4ee5\u4e0b yaml \u6587\u4ef6\uff0c\u90e8\u7f72 Prometheus Server\uff1a</p> <pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nlabels:\napp: prometheus\nname: prometheus\nnamespace: default\nspec:\nreplicas: 1\nselector:\nmatchLabels:\napp: prometheus\nstrategy:\nrollingUpdate:\nmaxSurge: 25%\nmaxUnavailable: 25%\ntype: RollingUpdate\ntemplate:\nmetadata:\nlabels:\napp: prometheus\nspec:\nserviceAccountName: prometheus\nserviceAccount: prometheus\ncontainers:\n- image: docker.io/prom/prometheus:latest\nimagePullPolicy: IfNotPresent\nname: prometheus\ncommand:\n- \"/bin/prometheus\"\nargs:\n- \"--config.file=/etc/prometheus/prometheus.yml\"\nports:\n- containerPort: 9090\nprotocol: TCP\nvolumeMounts:\n- mountPath: \"/etc/prometheus\"\nname: prometheus-config\nvolumes:\n- name: prometheus-config\nconfigMap:\nname: prometheus-config\n</code></pre> <p>\u5728\u90e8\u7f72\u5b8c Prometheus \u4e4b\u540e\uff0c\u53c2\u8003\u4ee5\u4e0b yaml \u6587\u4ef6\uff0c\u90e8\u7f72 Prometheus Service\uff1a</p> <pre><code>kind: Service\napiVersion: v1\nmetadata:\nname: prometheus\nnamespace: default\nlabels:\nname: prometheus\nspec:\nports:\n- name: test\nprotocol: TCP\nport: 9090\ntargetPort: 9090\ntype: NodePort\nselector:\napp: prometheus\nsessionAffinity: None\n</code></pre> <p>\u5c06 Prometheus \u901a\u8fc7 NodePort \u66b4\u9732\u540e\uff0c\u5373\u53ef\u901a\u8fc7\u8282\u70b9\u6765\u8bbf\u95ee Prometheus\u3002</p>"},{"location":"guide/prometheus/#prometheus_4","title":"Prometheus \u76d1\u63a7\u6570\u636e\u9a8c\u8bc1","text":"<p>\u67e5\u770b\u73af\u5883\u4e0a Prometheus \u76f8\u5173\u7684\u4fe1\u606f\uff1a</p> <pre><code># kubectl get svc \nNAME         TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)          AGE\nkubernetes   ClusterIP   10.4.0.1       &lt;none&gt;        443/TCP          8d\nprometheus   NodePort    10.4.102.222   &lt;none&gt;        9090:32611/TCP   8d\n# kubectl get pod -o wide\nNAME                          READY   STATUS    RESTARTS   AGE    IP          NODE              NOMINATED NODE   READINESS GATES\nprometheus-7544b6b84d-v9m8s   1/1     Running   0          3d5h   10.3.0.7    192.168.137.219   &lt;none&gt;           &lt;none&gt;\n# kubectl get endpoints -o wide\nNAME         ENDPOINTS                                                        AGE\nkubernetes   192.168.136.228:6443,192.168.136.232:6443,192.168.137.219:6443   8d\nprometheus   10.3.0.7:9090                                                    8d\n</code></pre> <p>\u901a\u8fc7 NodePort \u8bbf\u95ee Prometheus\uff0c\u67e5\u770b Status/Service Discovery \u52a8\u6001\u67e5\u8be2\u5230\u7684\u6570\u636e\uff1a</p> <p></p> <p>\u53ef\u4ee5\u770b\u5230\u5f53\u524d\u53ef\u4ee5\u67e5\u8be2\u5230\u96c6\u7fa4\u4e0a\u5168\u90e8\u7684 Service \u6570\u636e\u4fe1\u606f\u3002</p>"},{"location":"guide/prometheus/#_2","title":"\u914d\u7f6e\u67e5\u8be2\u6307\u5b9a\u7684\u8d44\u6e90","text":"<p>\u4ee5\u4e0a\u7684 ConfigMap \u914d\u7f6e\u4e2d\uff0c\u6ca1\u6709\u6dfb\u52a0\u8fc7\u6ee4\u6761\u4ef6\uff0c\u67e5\u8be2\u4e86\u6240\u6709\u7684\u8d44\u6e90\u6570\u636e\u3002\u5982\u679c\u53ea\u9700\u8981\u67d0\u4e2a\u89d2\u8272\u7684\u8d44\u6e90\u6570\u636e\uff0c\u5219\u53ef\u4ee5\u6dfb\u52a0\u8fc7\u6ee4\u6761\u4ef6\u3002</p> <p>\u4ee5 Service \u4e3a\u4f8b\uff0c\u4fee\u6539 ConfigMap \u5185\u5bb9\uff0c\u53ea\u67e5\u8be2\u5173\u5fc3\u7684 Service \u76d1\u63a7\u6570\u636e\u3002</p> <pre><code>    - job_name: 'kubernetes-service'\ntls_config:\nca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\nbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\nkubernetes_sd_configs:\n- role: service\nrelabel_configs:\n- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]\naction: \"keep\"\nregex: \"true\"\n- action: labelmap\nregex: __meta_kubernetes_service_label_(.+)\n- source_labels: [__meta_kubernetes_namespace]\ntarget_label: kubernetes_namespace\n- source_labels: [__meta_kubernetes_service_name]\ntarget_label: kubernetes_service_name\n- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]\naction: replace\ntarget_label: __metrics_path__\nregex: \"(.+)\"\n</code></pre> <p>Service \u9ed8\u8ba4\u76d1\u63a7\u8def\u5f84\u4e3a /metrics\u3002\u5982\u679c Service \u63d0\u4f9b\u7684\u76d1\u63a7\u6307\u6807\u662f\u5176\u4ed6\u7684\u8def\u5f84\uff0c\u53ef\u4ee5\u901a\u8fc7\u7ed9 Service \u6dfb\u52a0 annotation <code>prometheus.io/path</code> \u6765\u6307\u5b9a\u91c7\u96c6\u8def\u5f84\u3002</p> <p>\u5e94\u7528\u4ee5\u4e0a yaml\uff0c\u66f4\u65b0 ConfigMap \u4fe1\u606f\uff0c\u91cd\u5efa Prometheus Pod\uff0c\u4f7f\u914d\u7f6e\u751f\u6548\u3002</p> <p>\u67e5\u770b kube-system Namespace \u4e0b\u7684 Service \u4fe1\u606f\uff1a</p> <pre><code># kubectl get svc -n kube-system\nNAME                  TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)                  AGE\nkube-dns              ClusterIP   10.4.0.10      &lt;none&gt;        53/UDP,53/TCP,9153/TCP   13d\nkube-ovn-cni          ClusterIP   10.4.228.60    &lt;none&gt;        10665/TCP                13d\nkube-ovn-controller   ClusterIP   10.4.172.213   &lt;none&gt;        10660/TCP                13d\nkube-ovn-monitor      ClusterIP   10.4.242.9     &lt;none&gt;        10661/TCP                13d\nkube-ovn-pinger       ClusterIP   10.4.122.52    &lt;none&gt;        8080/TCP                 13d\novn-nb                ClusterIP   10.4.80.213    &lt;none&gt;        6641/TCP                 13d\novn-northd            ClusterIP   10.4.126.234   &lt;none&gt;        6643/TCP                 13d\novn-sb                ClusterIP   10.4.216.249   &lt;none&gt;        6642/TCP                 13d\n</code></pre> <p>\u7ed9 Service \u6dfb\u52a0 annotation <code>prometheus.io/scrape=\"true\"</code>\uff1a</p> <pre><code># kubectl annotate svc -n kube-system kube-ovn-cni  prometheus.io/scrape=true\nservice/kube-ovn-cni annotated\n# kubectl annotate svc -n kube-system kube-ovn-controller  prometheus.io/scrape=true\nservice/kube-ovn-controller annotated\n# kubectl annotate svc -n kube-system kube-ovn-monitor  prometheus.io/scrape=true\nservice/kube-ovn-monitor annotated\n# kubectl annotate svc -n kube-system kube-ovn-pinger  prometheus.io/scrape=true\nservice/kube-ovn-pinger annotated\n</code></pre> <p>\u67e5\u770b\u914d\u7f6e\u540e\u7684 Service \u4fe1\u606f\uff1a</p> <pre><code># kubectl get svc -o yaml -n kube-system kube-ovn-controller\napiVersion: v1\nkind: Service\nmetadata:\n  annotations:\n    helm.sh/chart-version: v3.10.0-alpha.55\n    helm.sh/original-name: kube-ovn-controller\n    ovn.kubernetes.io/vpc: ovn-cluster\n    prometheus.io/scrape: \"true\"                        // \u6dfb\u52a0\u7684 annotation\n  labels:\n    app: kube-ovn-controller\n  name: kube-ovn-controller\n  namespace: kube-system\nspec:\n  clusterIP: 10.4.172.213\n  clusterIPs:\n  - 10.4.172.213\n  internalTrafficPolicy: Cluster\n  ipFamilies:\n  - IPv4\n  ipFamilyPolicy: SingleStack\n  ports:\n  - name: metrics\n    port: 10660\nprotocol: TCP\n    targetPort: 10660\nselector:\n    app: kube-ovn-controller\n  sessionAffinity: None\n  type: ClusterIP\nstatus:\n  loadBalancer: {}\n</code></pre> <p>\u67e5\u770b Prometheus Status Targets \u4fe1\u606f\uff0c\u53ef\u4ee5\u770b\u5230\u53ea\u6709\u6dfb\u52a0\u4e86 annotation \u7684 Service \u88ab\u8fc7\u6ee4\u51fa\u6765\uff1a </p> <p>\u66f4\u591a\u5173\u4e8e relabel \u6dfb\u52a0\u8fc7\u6ee4\u53c2\u6570\u7684\u4fe1\u606f\uff0c\u53ef\u4ee5\u53c2\u8003 Prometheus-Relabel\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/qos/","title":"\u5bb9\u5668\u7f51\u7edc QoS \u914d\u7f6e","text":"<p>Kube-OVN \u652f\u6301\u57fa\u4e8e\u5355\u4e2a Pod \u7684\u4e24\u79cd\u4e0d\u540c\u7c7b\u578b\u7684 QoS\uff1a</p> <ul> <li>\u6700\u5927\u5e26\u5bbd\u9650\u5236 QoS\u3002</li> <li><code>linux-netem</code>\uff0c\u6a21\u62df\u8bbe\u5907\u5e72\u6270\u4e22\u5305\u7b49\u7684 QoS\uff0c\u53ef\u7528\u4e8e\u6a21\u62df\u6d4b\u8bd5\u3002</li> </ul> <p>\u76ee\u524d\u53ea\u652f\u6301 Pod \u7ea7\u522b QoS \u4e0d\u652f\u6301 Namespace \u6216 Subnet \u7ea7\u522b\u7684 QoS \u9650\u5236\u3002</p>"},{"location":"guide/qos/#qos_1","title":"\u57fa\u4e8e\u6700\u5927\u5e26\u5bbd\u9650\u5236\u7684 QoS","text":"<p>\u8be5\u7c7b\u578b\u7684 QoS \u53ef\u4ee5\u901a\u8fc7 Pod annotation \u52a8\u6001\u8fdb\u884c\u914d\u7f6e\uff0c\u53ef\u4ee5\u5728\u4e0d\u4e2d\u65ad Pod \u8fd0\u884c\u7684\u60c5\u51b5\u4e0b\u8fdb\u884c\u8c03\u6574\u3002 \u5e26\u5bbd\u9650\u901f\u7684\u5355\u4f4d\u4e3a <code>Mbit/s</code>\u3002</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: qos\nnamespace: ls1\nannotations:\novn.kubernetes.io/ingress_rate: \"3\"\novn.kubernetes.io/egress_rate: \"1\"\nspec:\ncontainers:\n- name: qos\nimage: docker.io/library/nginx:alpine\n</code></pre> <p>\u4f7f\u7528 annotation \u52a8\u6001\u8c03\u6574 QoS\uff1a</p> <pre><code>kubectl annotate --overwrite  pod nginx-74d5899f46-d7qkn ovn.kubernetes.io/ingress_rate=3\n</code></pre>"},{"location":"guide/qos/#qos_2","title":"\u6d4b\u8bd5 QoS \u8c03\u6574","text":"<p>\u90e8\u7f72\u6027\u80fd\u6d4b\u8bd5\u9700\u8981\u7684\u5bb9\u5668\uff1a</p> <pre><code>kind: DaemonSet\napiVersion: apps/v1\nmetadata:\nname: perf\nnamespace: ls1\nlabels:\napp: perf\nspec:\nselector:\nmatchLabels:\napp: perf\ntemplate:\nmetadata:\nlabels:\napp: perf\nspec:\ncontainers:\n- name: nginx\nimage: docker.io/kubeovn/perf\n</code></pre> <p>\u8fdb\u5165\u5176\u4e2d\u4e00\u4e2a Pod \u5e76\u5f00\u542f iperf3 server\uff1a</p> <pre><code># kubectl exec -it perf-4n4gt -n ls1 sh\n# iperf3 -s\n-----------------------------------------------------------\nServer listening on 5201\n-----------------------------------------------------------\n</code></pre> <p>\u8fdb\u5165\u53e6\u4e00\u4e2a Pod \u8bf7\u6c42\u4e4b\u524d\u7684 Pod\uff1a</p> <pre><code># kubectl exec -it perf-d4mqc -n ls1 sh\n# iperf3 -c 10.66.0.12\nConnecting to host 10.66.0.12, port 5201\n[  4] local 10.66.0.14 port 51544 connected to 10.66.0.12 port 5201\n[ ID] Interval           Transfer     Bandwidth       Retr  Cwnd\n[  4]   0.00-1.00   sec  86.4 MBytes   725 Mbits/sec    3    350 KBytes\n[  4]   1.00-2.00   sec  89.9 MBytes   754 Mbits/sec  118    473 KBytes\n[  4]   2.00-3.00   sec   101 MBytes   848 Mbits/sec  184    586 KBytes\n[  4]   3.00-4.00   sec   104 MBytes   875 Mbits/sec  217    671 KBytes\n[  4]   4.00-5.00   sec   111 MBytes   935 Mbits/sec  175    772 KBytes\n[  4]   5.00-6.00   sec   100 MBytes   840 Mbits/sec  658    598 KBytes\n[  4]   6.00-7.00   sec   106 MBytes   890 Mbits/sec  742    668 KBytes\n[  4]   7.00-8.00   sec   102 MBytes   857 Mbits/sec  764    724 KBytes\n[  4]   8.00-9.00   sec  97.4 MBytes   817 Mbits/sec  1175    764 KBytes\n[  4]   9.00-10.00  sec   111 MBytes   934 Mbits/sec  1083    838 KBytes\n- - - - - - - - - - - - - - - - - - - - - - - - -\n[ ID] Interval           Transfer     Bandwidth       Retr\n[  4]   0.00-10.00  sec  1010 MBytes   848 Mbits/sec  5119             sender\n[  4]   0.00-10.00  sec  1008 MBytes   846 Mbits/sec                  receiver\n\niperf Done.\n</code></pre> <p>\u4fee\u6539\u7b2c\u4e00\u4e2a Pod \u7684\u5165\u53e3\u5e26\u5bbd QoS\uff1a</p> <pre><code>kubectl annotate --overwrite  pod perf-4n4gt -n ls1 ovn.kubernetes.io/ingress_rate=30\n</code></pre> <p>\u518d\u6b21\u4ece\u7b2c\u4e8c\u4e2a Pod \u6d4b\u8bd5\u7b2c\u4e00\u4e2a Pod \u5e26\u5bbd\uff1a</p> <pre><code># iperf3 -c 10.66.0.12\nConnecting to host 10.66.0.12, port 5201\n[  4] local 10.66.0.14 port 52372 connected to 10.66.0.12 port 5201\n[ ID] Interval           Transfer     Bandwidth       Retr  Cwnd\n[  4]   0.00-1.00   sec  3.66 MBytes  30.7 Mbits/sec    2   76.1 KBytes\n[  4]   1.00-2.00   sec  3.43 MBytes  28.8 Mbits/sec    0    104 KBytes\n[  4]   2.00-3.00   sec  3.50 MBytes  29.4 Mbits/sec    0    126 KBytes\n[  4]   3.00-4.00   sec  3.50 MBytes  29.3 Mbits/sec    0    144 KBytes\n[  4]   4.00-5.00   sec  3.43 MBytes  28.8 Mbits/sec    0    160 KBytes\n[  4]   5.00-6.00   sec  3.43 MBytes  28.8 Mbits/sec    0    175 KBytes\n[  4]   6.00-7.00   sec  3.50 MBytes  29.3 Mbits/sec    0    212 KBytes\n[  4]   7.00-8.00   sec  3.68 MBytes  30.9 Mbits/sec    0    294 KBytes\n[  4]   8.00-9.00   sec  3.74 MBytes  31.4 Mbits/sec    0    398 KBytes\n[  4]   9.00-10.00  sec  3.80 MBytes  31.9 Mbits/sec    0    526 KBytes\n- - - - - - - - - - - - - - - - - - - - - - - - -\n[ ID] Interval           Transfer     Bandwidth       Retr\n[  4]   0.00-10.00  sec  35.7 MBytes  29.9 Mbits/sec    2             sender\n[  4]   0.00-10.00  sec  34.5 MBytes  29.0 Mbits/sec                  receiver\n\niperf Done.\n</code></pre>"},{"location":"guide/qos/#linux-netem-qos","title":"linux-netem QoS","text":"<p>Pod \u53ef\u4ee5\u4f7f\u7528\u5982\u4e0b annotation \u914d\u7f6e <code>linux-netem</code> \u7c7b\u578b QoS\uff1a <code>ovn.kubernetes.io/latency</code>\u3001<code>ovn.kubernetes.io/limit</code> \u548c <code>ovn.kubernetes.io/loss</code>\u3002</p> <ul> <li><code>ovn.kubernetes.io/latency</code>\uff1a\u8bbe\u7f6e Pod \u6d41\u91cf\u5ef6\u8fdf\uff0c\u53d6\u503c\u4e3a\u6574\u6570\uff0c\u5355\u4f4d\u4e3a ms\u3002</li> <li><code>ovn.kubernetes.io/limit</code>\uff1a \u4e3a <code>qdisc</code> \u961f\u5217\u53ef\u5bb9\u7eb3\u7684\u6700\u5927\u6570\u636e\u5305\u6570\uff0c\u53d6\u503c\u4e3a\u6574\u5f62\u6570\u503c\uff0c\u4f8b\u5982 1000\u3002</li> <li><code>ovn.kubernetes.io/loss</code>\uff1a \u4e3a\u8bbe\u7f6e\u7684\u62a5\u6587\u4e22\u5305\u6982\u7387\uff0c\u53d6\u503c\u4e3a float \u7c7b\u578b\uff0c\u4f8b\u5982\u53d6\u503c\u4e3a 20\uff0c\u5219\u4e3a\u8bbe\u7f6e 20% \u7684\u4e22\u5305\u6982\u7387\u3002</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/setup-options/","title":"\u5b89\u88c5\u548c\u914d\u7f6e\u9009\u9879","text":"<p>\u5728\u4e00\u952e\u5b89\u88c5\u4e2d\u6211\u4eec\u4f7f\u7528\u9ed8\u8ba4\u914d\u7f6e\u8fdb\u884c\u5b89\u88c5\uff0cKube-OVN \u8fd8\u652f\u6301\u66f4\u591a \u81ea\u5b9a\u4e49\u914d\u7f6e\uff0c\u53ef\u5728\u5b89\u88c5\u811a\u672c\u4e2d\u8fdb\u884c\u914d\u7f6e\uff0c\u6216\u8005\u4e4b\u540e\u66f4\u6539\u5404\u4e2a\u7ec4\u4ef6\u7684\u53c2\u6570\u6765\u8fdb\u884c\u914d\u7f6e\u3002\u672c\u6587\u6863\u5c06\u4f1a\u4ecb\u7ecd\u8fd9\u4e9b\u81ea\u5b9a\u4e49\u9009\u9879 \u7684\u4f5c\u7528\uff0c\u4ee5\u53ca\u5982\u4f55\u8fdb\u884c\u914d\u7f6e\u3002</p>"},{"location":"guide/setup-options/#_2","title":"\u5185\u7f6e\u7f51\u7edc\u8bbe\u7f6e","text":"<p>Kube-OVN \u5728\u5b89\u88c5\u65f6\u4f1a\u914d\u7f6e\u4e24\u4e2a\u5185\u7f6e\u5b50\u7f51\uff1a</p> <ol> <li><code>default</code> \u5b50\u7f51\uff0c\u4f5c\u4e3a Pod \u5206\u914d IP \u4f7f\u7528\u7684\u9ed8\u8ba4\u5b50\u7f51\uff0c\u9ed8\u8ba4 CIDR \u4e3a <code>10.16.0.0/16</code>\uff0c\u7f51\u5173\u4e3a <code>10.16.0.1</code>\u3002</li> <li><code>join</code> \u5b50\u7f51\uff0c\u4f5c\u4e3a Node \u548c Pod \u4e4b\u95f4\u8fdb\u884c\u7f51\u7edc\u901a\u4fe1\u7684\u7279\u6b8a\u5b50\u7f51, \u9ed8\u8ba4 CIDR \u4e3a <code>100.64.0.0/16</code>\uff0c\u7f51\u5173\u4e3a <code>100.64.0.1</code>\u3002</li> </ol> <p>\u5728\u5b89\u88c5\u65f6\u53ef\u4ee5\u901a\u8fc7\u5b89\u88c5\u811a\u672c\u5185\u7684\u914d\u7f6e\u8fdb\u884c\u66f4\u6539\uff1a</p> <pre><code>POD_CIDR=\"10.16.0.0/16\"\nPOD_GATEWAY=\"10.16.0.1\"\nJOIN_CIDR=\"100.64.0.0/16\"\nEXCLUDE_IPS=\"\"\n</code></pre> <p><code>EXCLUDE_IP</code> \u53ef\u8bbe\u7f6e <code>POD_CIDR</code> \u4e0d\u8fdb\u884c\u5206\u914d\u7684\u5730\u5740\u8303\u56f4\uff0c\u683c\u5f0f\u4e3a\uff1a<code>192.168.10.20..192.168.10.30</code>\u3002</p> <p>\u9700\u8981\u6ce8\u610f Overlay \u60c5\u51b5\u4e0b\u8fd9\u4e24\u4e2a\u7f51\u7edc\u4e0d\u80fd\u548c\u5df2\u6709\u7684\u4e3b\u673a\u7f51\u7edc\u548c Service CIDR \u51b2\u7a81\u3002</p> <p>\u5728\u5b89\u88c5\u540e\u53ef\u4ee5\u5bf9\u8fd9\u4e24\u4e2a\u7f51\u7edc\u7684\u5730\u5740\u8303\u56f4\u8fdb\u884c\u4fee\u6539\u8bf7\u53c2\u8003\u4fee\u6539\u9ed8\u8ba4\u5b50\u7f51\u548c\u4fee\u6539 Join \u5b50\u7f51\u3002</p>"},{"location":"guide/setup-options/#service","title":"Service \u7f51\u6bb5\u914d\u7f6e","text":"<p>\u7531\u4e8e\u90e8\u5206 <code>kube-proxy</code> \u8bbe\u7f6e\u7684 iptables \u548c\u8def\u7531\u89c4\u5219\u4f1a\u548c Kube-OVN \u8bbe\u7f6e\u7684\u89c4\u5219\u4ea7\u751f\u4ea4\u96c6\uff0c\u56e0\u6b64 Kube-OVN \u9700\u8981\u77e5\u9053 Service \u7684 CIDR \u6765\u6b63\u786e\u8bbe\u7f6e\u5bf9\u5e94\u7684\u89c4\u5219\u3002</p> <p>\u5728\u5b89\u88c5\u811a\u672c\u4e2d\u53ef\u4ee5\u901a\u8fc7\u4fee\u6539\uff1a</p> <pre><code>SVC_CIDR=\"10.96.0.0/12\"  </code></pre> <p>\u6765\u8fdb\u884c\u914d\u7f6e\u3002</p> <p>\u4e5f\u53ef\u4ee5\u5728\u5b89\u88c5\u540e\u901a\u8fc7\u4fee\u6539 <code>kube-ovn-controller</code> Deployment \u7684\u53c2\u6570\uff1a</p> <pre><code>args:\n- --service-cluster-ip-range=10.96.0.0/12\n</code></pre> <p>\u6765\u8fdb\u884c\u4fee\u6539\u3002</p>"},{"location":"guide/setup-options/#overlay","title":"Overlay \u7f51\u5361\u9009\u62e9","text":"<p>\u5728\u8282\u70b9\u5b58\u5728\u591a\u5757\u7f51\u5361\u7684\u60c5\u51b5\u4e0b\uff0cKube-OVN \u9ed8\u8ba4\u4f1a\u9009\u62e9 Kubernetes Node IP \u5bf9\u5e94\u7684\u7f51\u5361\u4f5c\u4e3a\u5bb9\u5668\u95f4\u8de8\u8282\u70b9\u901a\u4fe1\u7684\u7f51\u5361\u5e76\u5efa\u7acb\u5bf9\u5e94\u7684\u96a7\u9053\u3002</p> <p>\u5982\u679c\u9700\u8981\u9009\u62e9\u5176\u4ed6\u7684\u7f51\u5361\u5efa\u7acb\u5bb9\u5668\u96a7\u9053\uff0c\u53ef\u4ee5\u5728\u5b89\u88c5\u811a\u672c\u4e2d\u4fee\u6539\uff1a</p> <pre><code>IFACE=eth1\n</code></pre> <p>\u8be5\u9009\u9879\u652f\u6301\u4ee5\u9017\u53f7\u6240\u5206\u9694\u6b63\u5219\u8868\u8fbe\u5f0f,\u4f8b\u5982<code>ens[a-z0-9]*,eth[a-z0-9]*</code>\u3002</p> <p>\u5b89\u88c5\u540e\u4e5f\u53ef\u901a\u8fc7\u4fee\u6539 <code>kube-ovn-cni</code> DaemonSet \u7684\u53c2\u6570\u8fdb\u884c\u8c03\u6574\uff1a</p> <pre><code>args:\n- --iface=eth1\n</code></pre> <p>\u5982\u679c\u6bcf\u53f0\u673a\u5668\u7684\u7f51\u5361\u540d\u5747\u4e0d\u540c\uff0c\u4e14\u6ca1\u6709\u56fa\u5b9a\u89c4\u5f8b\uff0c\u53ef\u4ee5\u4f7f\u7528\u8282\u70b9 annotation <code>ovn.kubernetes.io/tunnel_interface</code> \u8fdb\u884c\u6bcf\u4e2a\u8282\u70b9\u7684\u9010\u4e00\u914d\u7f6e\uff0c\u62e5\u6709\u8be5 annotation \u8282\u70b9\u4f1a\u8986\u76d6 <code>iface</code> \u7684\u914d\u7f6e\uff0c\u4f18\u5148\u4f7f\u7528 annotation\u3002</p> <pre><code>kubectl annotate node no1 ovn.kubernetes.io/tunnel_interface=ethx\n</code></pre>"},{"location":"guide/setup-options/#mtu","title":"MTU \u8bbe\u7f6e","text":"<p>\u7531\u4e8e Overlay \u5c01\u88c5\u9700\u8981\u5360\u636e\u989d\u5916\u7684\u7a7a\u95f4\uff0cKube-OVN \u5728\u521b\u5efa\u5bb9\u5668\u7f51\u5361\u65f6\u4f1a\u6839\u636e\u9009\u62e9\u7f51\u5361\u7684 MTU \u8fdb\u884c\u5bb9\u5668\u7f51\u5361\u7684 MTU \u8c03\u6574\uff0c \u9ed8\u8ba4\u60c5\u51b5\u4e0b Overlay \u5b50\u7f51\u4e0b Pod \u7f51\u5361 MTU \u4e3a\u4e3b\u673a\u7f51\u5361 MTU - 100\uff0cUnderlay \u5b50\u7f51\u4e0b\uff0cPod \u7f51\u5361\u548c\u4e3b\u673a\u7f51\u5361\u6709\u76f8\u540c MTU\u3002</p> <p>\u5982\u679c\u9700\u8981\u8c03\u6574 Overlay \u5b50\u7f51\u4e0b MTU \u7684\u5927\u5c0f\uff0c\u53ef\u4ee5\u4fee\u6539 <code>kube-ovn-cni</code> DaemonSet \u7684\u53c2\u6570\uff1a</p> <pre><code>args:\n- --mtu=1333\n</code></pre>"},{"location":"guide/setup-options/#_3","title":"\u5168\u5c40\u6d41\u91cf\u955c\u50cf\u5f00\u542f\u8bbe\u7f6e","text":"<p>\u5728\u5f00\u542f\u5168\u5c40\u6d41\u91cf\u955c\u50cf\u7684\u60c5\u51b5\u4e0b\uff0cKube-OVN \u4f1a\u5728\u6bcf\u4e2a\u8282\u70b9\u4e0a\u521b\u5efa\u4e00\u5757 <code>mirror0</code> \u7684\u865a\u62df\u7f51\u5361\uff0c\u590d\u5236\u5f53\u524d\u673a\u5668\u6240\u6709\u5bb9\u5668\u7f51\u7edc\u6d41\u91cf\u5230\u8be5\u7f51\u5361\u4e0a\uff0c \u7528\u6237\u53ef\u4ee5\u901a\u8fc7 tcpdump \u53ca\u5176\u4ed6\u5de5\u5177\u8fdb\u884c\u6d41\u91cf\u5206\u6790\uff0c\u8be5\u529f\u80fd\u53ef\u4ee5\u5728\u5b89\u88c5\u811a\u672c\u4e2d\u901a\u8fc7\u4e0b\u9762\u7684\u914d\u7f6e\u5f00\u542f\uff1a</p> <pre><code>ENABLE_MIRROR=true\n</code></pre> <p>\u4e5f\u53ef\u5728\u5b89\u88c5\u540e\u901a\u8fc7\u4fee\u6539 <code>kube-ovn-cni</code> DaemonSet \u7684\u53c2\u6570\u65b9\u5f0f\u8fdb\u884c\u8c03\u6574:</p> <pre><code>args:\n- --enable-mirror=true\n</code></pre> <p>\u6d41\u91cf\u955c\u50cf\u7684\u80fd\u529b\u5728\u9ed8\u8ba4\u5b89\u88c5\u4e2d\u4e3a\u5173\u95ed\uff0c\u5982\u679c\u9700\u8981\u7ec6\u7c92\u5ea6\u7684\u6d41\u91cf\u955c\u50cf\u6216\u9700\u8981\u5c06\u6d41\u91cf\u955c\u50cf\u5230\u989d\u5916\u7684\u7f51\u5361\u8bf7\u53c2\u8003\u5bb9\u5668\u7f51\u7edc\u6d41\u91cf\u955c\u50cf\u3002</p>"},{"location":"guide/setup-options/#lb","title":"LB \u5f00\u542f\u8bbe\u7f6e","text":"<p>Kube-OVN \u4f7f\u7528 OVN \u4e2d\u7684 L2 LB \u6765\u5b9e\u73b0 Service \u8f6c\u53d1\uff0c\u5728 Overlay \u573a\u666f\u4e2d\uff0c\u7528\u6237\u53ef\u4ee5\u9009\u62e9\u4f7f\u7528 <code>kube-proxy</code> \u6765\u5b8c\u6210 Service \u6d41\u91cf\u8f6c\u53d1, \u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\u53ef\u4ee5\u5173\u95ed Kube-OVN \u7684 LB \u529f\u80fd\u4ee5\u8fbe\u5230\u63a7\u5236\u9762\u548c\u6570\u636e\u9762\u66f4\u597d\u7684\u6027\u80fd\u3002</p> <p>\u8be5\u529f\u80fd\u53ef\u4ee5\u5728\u5b89\u88c5\u811a\u672c\u4e2d\u8fdb\u884c\u914d\u7f6e\uff1a</p> <pre><code>ENABLE_LB=false\n</code></pre> <p>\u6216\u8005\u5728\u5b89\u88c5\u540e\u901a\u8fc7\u66f4\u6539 <code>kube-ovn-controller</code> Deployment \u7684\u53c2\u6570\u8fdb\u884c\u914d\u7f6e\uff1a</p> <pre><code>args:\n- --enable-lb=false\n</code></pre> <p>LB \u7684\u529f\u80fd\u5728\u9ed8\u8ba4\u5b89\u88c5\u4e2d\u4e3a\u5f00\u542f\u3002</p> <p>\u4ece Kube-OVN v1.12.0 \u7248\u672c\u5f00\u59cb\uff0c\u5728 subnet crd \u5b9a\u4e49\u4e2d\u589e\u52a0\u4e86 spec \u5b57\u6bb5 <code>enableLb</code>\uff0c\u5c06 Kube-OVN \u7684 LB \u529f\u80fd\u8fc1\u79fb\u5230\u5b50\u7f51\u5c42\u7ea7\uff0c\u53ef\u4ee5\u57fa\u4e8e\u4e0d\u540c\u7684\u5b50\u7f51\u5206\u522b\u8bbe\u7f6e\u662f\u5426\u5f00\u542f LB \u529f\u80fd\u3002<code>kube-ovn-controller</code> Deployment \u4e2d\u7684 <code>enable-lb</code> \u53c2\u6570\u4f5c\u4e3a\u5168\u5c40\u53c2\u6570\uff0c\u63a7\u5236\u662f\u5426\u521b\u5efa load-balancer \u8bb0\u5f55\uff0c\u5b50\u7f51\u4e2d\u65b0\u589e\u7684 <code>enableLb</code> \u53c2\u6570\u7528\u4e8e\u63a7\u5236\u5b50\u7f51\u662f\u5426\u5173\u8054 load-balancer \u8bb0\u5f55\u3002\u4e4b\u524d\u7248\u672c\u5347\u7ea7\u5230 v1.12.0 \u4e4b\u540e\uff0c\u5b50\u7f51 <code>enableLb</code> \u53c2\u6570\u4f1a\u81ea\u52a8\u7ee7\u627f\u539f\u6709\u7684\u5168\u5c40\u5f00\u5173\u53c2\u6570\u53d6\u503c\u3002</p>"},{"location":"guide/setup-options/#networkpolicy","title":"NetworkPolicy \u5f00\u542f\u8bbe\u7f6e","text":"<p>Kube-OVN \u4f7f\u7528 OVN \u4e2d\u7684 ACL \u6765\u5b9e\u73b0 NetworkPolicy\uff0c\u7528\u6237\u53ef\u4ee5\u9009\u62e9\u5173\u95ed NetworkPolicy \u529f\u80fd \u6216\u8005\u4f7f\u7528 Cilium Chain \u7684\u65b9\u5f0f\u5229\u7528 eBPF \u5b9e\u73b0 NetworkPolicy\uff0c \u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\u53ef\u4ee5\u5173\u95ed Kube-OVN \u7684 NetworkPolicy \u529f\u80fd\u4ee5\u8fbe\u5230\u63a7\u5236\u9762\u548c\u6570\u636e\u9762\u66f4\u597d\u7684\u6027\u80fd\u3002</p> <p>\u8be5\u529f\u80fd\u53ef\u4ee5\u5728\u5b89\u88c5\u811a\u672c\u4e2d\u8fdb\u884c\u914d\u7f6e\uff1a</p> <pre><code>ENABLE_NP=false\n</code></pre> <p>\u6216\u8005\u5728\u5b89\u88c5\u540e\u901a\u8fc7\u66f4\u6539 <code>kube-ovn-controller</code> Deployment \u7684\u53c2\u6570\u8fdb\u884c\u914d\u7f6e\uff1a</p> <pre><code>args:\n- --enable-np=false\n</code></pre> <p>NetworkPolicy \u7684\u80fd\u529b\u5728\u9ed8\u8ba4\u5b89\u88c5\u4e2d\u4e3a\u5f00\u542f\u3002</p>"},{"location":"guide/setup-options/#eip-snat","title":"EIP \u548c SNAT \u5f00\u542f\u8bbe\u7f6e","text":"<p>\u9ed8\u8ba4\u7f51\u7edc\u4e0b\u5982\u679c\u65e0\u9700\u4f7f\u7528 EIP \u548c SNAT \u7684\u80fd\u529b\uff0c\u53ef\u4ee5\u9009\u62e9\u5173\u95ed\u76f8\u5173\u529f\u80fd\uff0c\u4ee5\u51cf\u5c11 <code>kube-ovn-controller</code> \u5728\u521b\u5efa\u548c\u66f4\u65b0 \u7f51\u7edc\u65f6\u7684\u68c0\u67e5\u6d88\u8017\uff0c\u5728\u5927\u89c4\u6a21\u96c6\u7fa4\u73af\u5883\u4e0b\u53ef\u4ee5\u63d0\u5347\u5904\u7406\u901f\u5ea6\u3002</p> <p>\u8be5\u529f\u80fd\u53ef\u5728\u5b89\u88c5\u811a\u672c\u4e2d\u8fdb\u884c\u914d\u7f6e\uff1a</p> <pre><code>ENABLE_EIP_SNAT=false\n</code></pre> <p>\u6216\u8005\u5728\u5b89\u88c5\u540e\u901a\u8fc7\u66f4\u6539 <code>kube-ovn-controller</code> Deployment \u7684\u53c2\u6570\u8fdb\u884c\u914d\u7f6e\uff1a</p> <pre><code>args:\n- --enable-eip-snat=false\n</code></pre> <p>EIP \u548c SNAT \u7684\u80fd\u529b\u5728\u9ed8\u8ba4\u5b89\u88c5\u4e2d\u4e3a\u5f00\u542f\u3002\u8be5\u529f\u80fd\u7684\u76f8\u5173\u4f7f\u7528\u548c\u5176\u4ed6\u53ef\u914d\u53c2\u6570\u8bf7\u53c2\u8003 EIP \u548c SNAT \u914d\u7f6e\u3002</p>"},{"location":"guide/setup-options/#load-balancer-service","title":"Load Balancer \u7c7b\u578b Service \u652f\u6301\u5f00\u542f\u8bbe\u7f6e","text":"<p>\u9ed8\u8ba4 VPC \u4e0b\u53ef\u901a\u8fc7\u5f00\u542f\u8be5\u9009\u9879\u6765\u652f\u6301 Load Balancer \u7c7b\u578b Service\u3002\u8be5\u529f\u80fd\u7684\u76f8\u5173\u4f7f\u7528\u548c\u5176\u4ed6\u53ef\u914d\u53c2\u6570\u8bf7\u53c2\u8003 LoadBalancer \u7c7b\u578b Service\u3002</p> <p>\u8be5\u529f\u80fd\u9ed8\u8ba4\u5173\u95ed\uff0c\u53ef\u5728\u5b89\u88c5\u811a\u672c\u4e2d\u8fdb\u884c\u914d\u7f6e\uff1a</p> <pre><code>ENABLE_LB_SVC=true\n</code></pre> <p>\u6216\u8005\u5728\u5b89\u88c5\u540e\u901a\u8fc7\u66f4\u6539 <code>kube-ovn-controller</code> Deployment \u7684\u53c2\u6570\u8fdb\u884c\u914d\u7f6e\uff1a</p> <pre><code>args:\n- --enable-lb-svc=true\n</code></pre>"},{"location":"guide/setup-options/#ecmp","title":"\u96c6\u4e2d\u5f0f\u7f51\u5173 ECMP \u5f00\u542f\u8bbe\u7f6e","text":"<p>\u96c6\u4e2d\u5f0f\u7f51\u5173\u652f\u6301\u4e3b\u5907\u548c ECMP \u4e24\u79cd\u9ad8\u53ef\u7528\u6a21\u5f0f\uff0c\u5982\u679c\u9700\u8981\u542f\u7528 ECMP \u6a21\u5f0f\uff0c \u9700\u8981\u66f4\u6539 <code>kube-ovn-controller</code> Deployment \u7684\u53c2\u6570\u8fdb\u884c\u914d\u7f6e:</p> <pre><code>args:\n- --enable-ecmp=true </code></pre> <p>\u4ece Kube-OVN v1.12.0 \u7248\u672c\u5f00\u59cb\uff0c\u5728 subnet crd \u5b9a\u4e49\u4e2d\u589e\u52a0\u4e86 spec \u5b57\u6bb5 <code>enableEcmp</code>\uff0c\u5c06\u96c6\u4e2d\u5f0f\u5b50\u7f51 ECMP \u5f00\u5173\u63a7\u5236\u8fc1\u79fb\u5230\u5b50\u7f51\u5c42\u7ea7\uff0c\u53ef\u4ee5\u57fa\u4e8e\u4e0d\u540c\u7684\u5b50\u7f51\u5206\u522b\u8bbe\u7f6e\u662f\u5426\u5f00\u542f ECMP \u6a21\u5f0f\u3002\u539f\u6709\u7684 <code>kube-ovn-controller</code> Deployment \u4e2d\u7684 <code>enable-ecmp</code> \u53c2\u6570\u4e0d\u518d\u4f7f\u7528\u3002\u4e4b\u524d\u7248\u672c\u5347\u7ea7\u5230 v1.12.0 \u4e4b\u540e\uff0c\u5b50\u7f51\u5f00\u5173\u4f1a\u81ea\u52a8\u7ee7\u627f\u539f\u6709\u7684\u5168\u5c40\u5f00\u5173\u53c2\u6570\u53d6\u503c\u3002</p> <p>\u96c6\u4e2d\u5f0f\u7f51\u5173\u9ed8\u8ba4\u5b89\u88c5\u4e0b\u4e3a\u4e3b\u5907\u6a21\u5f0f\uff0c\u66f4\u591a\u7f51\u5173\u76f8\u5173\u5185\u5bb9\u8bf7\u53c2\u8003\u5b50\u7f51\u4f7f\u7528\u3002</p>"},{"location":"guide/setup-options/#kubevirt-vm","title":"Kubevirt VM \u56fa\u5b9a\u5730\u5740\u5f00\u542f\u8bbe\u7f6e","text":"<p>\u9488\u5bf9 Kubevirt \u521b\u5efa\u7684 VM \u5b9e\u4f8b\uff0c<code>kube-ovn-controller</code> \u53ef\u4ee5\u6309\u7167\u7c7b\u4f3c StatefulSet Pod \u7684\u65b9\u5f0f\u8fdb\u884c IP \u5730\u5740\u5206\u914d\u548c\u7ba1\u7406\u3002 \u4ee5\u8fbe\u5230 VM \u5b9e\u4f8b\u5728\u751f\u547d\u5468\u671f\u5185\u542f\u505c\uff0c\u5347\u7ea7\uff0c\u8fc1\u79fb\u7b49\u64cd\u4f5c\u8fc7\u7a0b\u4e2d\u5730\u5740\u56fa\u5b9a\u4e0d\u53d8\uff0c\u66f4\u7b26\u865a\u62df\u5316\u5408\u7528\u6237\u7684\u5b9e\u9645\u4f7f\u7528\u4f53\u9a8c\u3002</p> <p>\u8be5\u529f\u80fd\u5728 1.10.6 \u540e\u9ed8\u8ba4\u5f00\u542f\uff0c\u82e5\u8981\u5173\u95ed\u6b64\u529f\u80fd\uff0c\u9700\u8981\u5728 <code>kube-ovn-controller</code> Deployment \u7684\u542f\u52a8\u547d\u4ee4\u4e2d\u8bbe\u7f6e\u5982\u4e0b\u53c2\u6570\uff1a</p> <pre><code>args:\n- --keep-vm-ip=false\n</code></pre>"},{"location":"guide/setup-options/#cni","title":"CNI \u914d\u7f6e\u76f8\u5173\u8bbe\u7f6e","text":"<p>Kube-OVN \u9ed8\u8ba4\u4f1a\u5728 <code>/opt/cni/bin</code> \u76ee\u5f55\u4e0b\u5b89\u88c5 CNI \u6267\u884c\u6587\u4ef6\uff0c\u5728 <code>/etc/cni/net.d</code> \u76ee\u5f55\u4e0b\u5b89\u88c5 CNI \u914d\u7f6e\u6587\u4ef6 <code>01-kube-ovn.conflist</code>\u3002 \u5982\u679c\u9700\u8981\u66f4\u6539\u5b89\u88c5\u4f4d\u7f6e\u548c CNI \u914d\u7f6e\u6587\u4ef6\u7684\u4f18\u5148\u7ea7\uff0c\u53ef\u4ee5\u901a\u8fc7\u5b89\u88c5\u811a\u672c\u7684\u4e0b\u5217\u53c2\u6570\u8fdb\u884c\u8c03\u6574\uff1a</p> <pre><code>CNI_CONF_DIR=\"/etc/cni/net.d\"\nCNI_BIN_DIR=\"/opt/cni/bin\"\nCNI_CONFIG_PRIORITY=\"01\"\n</code></pre> <p>\u6216\u8005\u5728\u5b89\u88c5\u540e\u66f4\u6539 <code>kube-ovn-cni</code> DaemonSet \u7684 Volume \u6302\u8f7d\u548c\u542f\u52a8\u53c2\u6570\uff1a</p> <pre><code>volumes:\n- name: cni-conf\nhostPath:\npath: \"/etc/cni/net.d\"\n- name: cni-bin\nhostPath:\npath:\"/opt/cni/bin\"\n...\nargs:\n- --cni-conf-name=01-kube-ovn.conflist\n</code></pre>"},{"location":"guide/setup-options/#_4","title":"\u96a7\u9053\u7c7b\u578b\u8bbe\u7f6e","text":"<p>Kube-OVN \u9ed8\u8ba4 Overlay \u7684\u5c01\u88c5\u6a21\u5f0f\u4e3a Geneve\uff0c\u5982\u679c\u60f3\u66f4\u6362\u4e3a Vxlan \u6216 STT\uff0c\u53ef\u4ee5\u901a\u8fc7\u5b89\u88c5\u811a\u672c\u7684\u4e0b\u5217\u53c2\u6570\u8fdb\u884c\u8c03\u6574\uff1a</p> <pre><code>TUNNEL_TYPE=\"vxlan\"\n</code></pre> <p>\u6216\u8005\u5728\u5b89\u88c5\u540e\u66f4\u6539 <code>ovs-ovn</code> DaemonSet \u7684\u73af\u5883\u53d8\u91cf\uff1a</p> <pre><code>env:\n- name: TUNNEL_TYPE\nvalue: \"vxlan\"\n</code></pre> <p>\u5982\u679c\u9700\u8981\u4f7f\u7528 STT \u96a7\u9053\u9700\u8981\u989d\u5916\u7f16\u8bd1 ovs \u7684\u5185\u6838\u6a21\u5757\uff0c\u8bf7\u53c2\u8003\u6027\u80fd\u8c03\u4f18\u3002</p> <p>\u4e0d\u540c\u534f\u8bae\u5728\u5b9e\u9645\u4f7f\u7528\u4e2d\u7684\u533a\u522b\u8bf7\u53c2\u8003\u96a7\u9053\u534f\u8bae\u8bf4\u660e\u3002</p>"},{"location":"guide/setup-options/#ssl","title":"SSL \u8bbe\u7f6e","text":"<p>OVN DB \u7684 API \u63a5\u53e3\u652f\u6301 SSL \u52a0\u5bc6\u6765\u4fdd\u8bc1\u8fde\u63a5\u5b89\u5168\uff0c\u5982\u8981\u5f00\u542f\u53ef\u8c03\u6574\u5b89\u88c5\u811a\u672c\u4e2d\u7684\u5982\u4e0b\u53c2\u6570:</p> <pre><code>ENABLE_SSL=true\n</code></pre> <p>SSL \u529f\u80fd\u9ed8\u8ba4\u5b89\u88c5\u4e0b\u4e3a\u5173\u95ed\u6a21\u5f0f\u3002</p>"},{"location":"guide/setup-options/#ip","title":"\u7ed1\u5b9a\u672c\u5730 ip","text":"<p>kube-ovn-controller/kube-ovn-cni/kube-ovn-monitor \u8fd9\u4e9b\u670d\u52a1\u652f\u6301\u7ed1\u5b9a\u672c\u5730 ip\uff0c\u8be5\u529f\u80fd\u8bbe\u8ba1\u539f\u56e0\u4e3b\u8981\u662f\u56e0\u4e3a\u67d0\u4e9b\u573a\u666f\u4e0b\u51fa\u4e8e\u5b89\u5168\u8003\u8651\u4e0d\u5141\u8bb8\u670d\u52a1\u7ed1\u5b9a 0.0.0.0 \uff08\u6bd4\u5982\u8be5\u670d\u52a1\u90e8\u7f72\u5728\u67d0\u4e2a\u5bf9\u5916\u7f51\u5173\u4e0a\uff0c\u5916\u90e8\u7528\u6237\u53ef\u4ee5\u76f4\u63a5\u901a\u8fc7\u516c\u7f51 ip \u5e76\u6307\u5b9a\u7aef\u53e3\u53bb\u8bbf\u95ee\u5230\u8be5\u670d\u52a1\uff09\uff0c\u8be5\u529f\u80fd\u9ed8\u8ba4\u662f\u6253\u5f00\u7684\uff0c\u7531\u5b89\u88c5\u811a\u672c\u4e2d\u5982\u4e0b\u53c2\u6570\u63a7\u5236\uff1a</p> <pre><code>ENABLE_BIND_LOCAL_IP=true\n</code></pre> <p>\u4ee5 kube-ovn-monitor \u4e3a\u4f8b\uff0c\u5f00\u542f\u529f\u80fd\u540e\u4f1a\u628a\u670d\u52a1\u7ed1\u5b9a\u672c\u5730\u7684 pod ip \u5982\u4e0b\uff1a</p> <pre><code># netstat -tunlp |grep kube-ovn\ntcp        0      0 172.18.0.5:10661        0.0.0.0:*               LISTEN      2612/./kube-ovn-mon\n</code></pre> <p>\u5b89\u88c5\u540e\u4e5f\u53ef\u901a\u8fc7\u4fee\u6539\u670d\u52a1\u7684 deployment \u6216\u8005 daemonSet \u7684\u73af\u5883\u53d8\u91cf\u53c2\u6570\u8fdb\u884c\u8c03\u6574\uff1a</p> <pre><code>env:\n- name: ENABLE_BIND_LOCAL_IP\nvalue: \"false\"\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/static-ip-mac/","title":"\u56fa\u5b9a\u5730\u5740","text":"<p>Kube-OVN \u9ed8\u8ba4\u4f1a\u6839\u636e Pod \u6240\u5728 Namespace \u6240\u5c5e\u7684\u5b50\u7f51\u4e2d\u968f\u673a\u5206\u914d IP \u548c Mac\u3002 \u9488\u5bf9\u5de5\u4f5c\u8d1f\u8f7d\u9700\u8981\u56fa\u5b9a\u5730\u5740\u7684\u60c5\u51b5\uff0cKube-OVN \u6839\u636e\u4e0d\u540c\u7684\u573a\u666f\uff0c\u63d0\u4f9b\u4e86\u591a\u79cd\u56fa\u5b9a\u5730\u5740\u7684\u65b9\u6cd5\uff1a</p> <ul> <li>\u5355\u4e2a Pod \u56fa\u5b9a IP/Mac\u3002</li> <li>Workload \u901a\u7528 IP Pool \u65b9\u5f0f\u6307\u5b9a\u56fa\u5b9a\u5730\u5740\u8303\u56f4\u3002</li> <li>StatefulSet \u56fa\u5b9a\u5730\u5740\u3002</li> <li>KubeVirt VM \u56fa\u5b9a\u5730\u5740\u3002</li> </ul>"},{"location":"guide/static-ip-mac/#pod-ip-mac","title":"\u5355\u4e2a Pod \u56fa\u5b9a IP \u548c Mac","text":"<p>\u53ef\u4ee5\u5728\u521b\u5efa Pod \u65f6\u901a\u8fc7 annotation \u6765\u6307\u5b9a Pod \u8fd0\u884c\u65f6\u6240\u9700\u7684 IP/Mac, <code>kube-ovn-controller</code> \u8fd0\u884c\u65f6\u5c06\u4f1a\u8df3\u8fc7\u5730\u5740\u968f\u673a\u5206\u914d\u9636\u6bb5\uff0c\u7ecf\u8fc7\u51b2\u7a81\u68c0\u6d4b\u540e\u76f4\u63a5\u4f7f\u7528\u6307\u5b9a\u5730\u5740\uff0c\u5982\u4e0b\u6240\u793a\uff1a</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: static-ip\nannotations:\novn.kubernetes.io/ip_address: 10.16.0.15   // \u53cc\u6808\u5730\u5740\u4f7f\u7528\u9017\u53f7\u5206\u9694 10.16.0.15,fd00:10:16::15\novn.kubernetes.io/mac_address: 00:00:00:53:6B:B6\nspec:\ncontainers:\n- name: static-ip\nimage: docker.io/library/nginx:alpine\n</code></pre> <p>\u5728\u4f7f\u7528 annotation \u5b9a\u4e49\u5355\u4e2a Pod IP/Mac \u65f6\u9700\u8981\u6ce8\u610f\u4ee5\u4e0b\u51e0\u70b9\uff1a</p> <ol> <li>\u6240\u4f7f\u7528\u7684 IP/Mac \u4e0d\u80fd\u548c\u5df2\u6709\u7684 IP/Mac \u51b2\u7a81\u3002</li> <li>IP \u5fc5\u987b\u5728\u6240\u5c5e\u5b50\u7f51\u7684 CIDR \u5185\u3002</li> <li>\u53ef\u4ee5\u53ea\u6307\u5b9a IP \u6216 Mac\uff0c\u53ea\u6307\u5b9a\u4e00\u4e2a\u65f6\uff0c\u53e6\u4e00\u4e2a\u4f1a\u968f\u673a\u5206\u914d\u3002</li> </ol>"},{"location":"guide/static-ip-mac/#workload-ip-pool","title":"Workload \u901a\u7528 IP Pool \u56fa\u5b9a\u5730\u5740","text":"<p>Kube-OVN \u652f\u6301\u901a\u8fc7 annotation <code>ovn.kubernetes.io/ip_pool</code> \u7ed9 Workload\uff08Deployment/StatefulSet/DaemonSet/Job/CronJob\uff09\u8bbe\u7f6e\u56fa\u5b9a IP\u3002 <code>kube-ovn-controller</code> \u4f1a\u81ea\u52a8\u9009\u62e9 <code>ovn.kubernetes.io/ip_pool</code> \u4e2d\u6307\u5b9a\u7684 IP \u5e76\u8fdb\u884c\u51b2\u7a81\u68c0\u6d4b\u3002</p> <p>IP Pool \u7684 Annotation \u9700\u8981\u52a0\u5728 <code>template</code> \u5185\u7684 <code>annotation</code> \u5b57\u6bb5\uff0c\u9664\u4e86 Kubernetes \u5185\u7f6e\u7684 Workload \u7c7b\u578b\uff0c \u5176\u4ed6\u7528\u6237\u81ea\u5b9a\u4e49\u7684 Workload \u4e5f\u53ef\u4ee5\u4f7f\u7528\u540c\u6837\u7684\u65b9\u5f0f\u8fdb\u884c\u56fa\u5b9a\u5730\u5740\u5206\u914d\u3002</p>"},{"location":"guide/static-ip-mac/#deployment-ip","title":"Deployment \u56fa\u5b9a IP \u793a\u4f8b","text":"<pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nname: ippool\nlabels:\napp: ippool\nspec:\nreplicas: 2\nselector:\nmatchLabels:\napp: ippool\ntemplate:\nmetadata:\nlabels:\napp: ippool\nannotations:\novn.kubernetes.io/ip_pool: 10.16.0.15,10.16.0.16,10.16.0.17 // \u53cc\u6808\u5730\u5740\u4f7f\u7528\u5206\u53f7\u8fdb\u884c\u5206\u9694 10.16.0.15,fd00:10:16::000E;10.16.0.16,fd00:10:16::000F;10.16.0.17,fd00:10:16::0010\nspec:\ncontainers:\n- name: ippool\nimage: docker.io/library/nginx:alpine\n</code></pre> <p>\u5bf9 Workload \u4f7f\u7528\u56fa\u5b9a IP \u9700\u8981\u6ce8\u610f\u4ee5\u4e0b\u51e0\u70b9\uff1a</p> <ol> <li><code>ovn.kubernetes.io/ip_pool</code> \u4e2d\u7684 IP \u5e94\u8be5\u5c5e\u4e8e\u6240\u5728\u5b50\u7f51\u7684 CIDR \u5185\u3002</li> <li><code>ovn.kubernetes.io/ip_pool</code> \u4e2d\u7684 IP \u4e0d\u80fd\u548c\u5df2\u4f7f\u7528\u7684 IP \u51b2\u7a81\u3002</li> <li>\u5f53 <code>ovn.kubernetes.io/ip_pool</code> \u4e2d\u7684 IP \u6570\u91cf\u5c0f\u4e8e replicas \u6570\u91cf\u65f6\uff0c\u591a\u51fa\u7684 Pod \u5c06\u65e0\u6cd5\u521b\u5efa\u3002\u4f60\u9700\u8981\u6839\u636e Workload \u7684\u66f4\u65b0\u7b56\u7565\u4ee5\u53ca\u6269\u5bb9\u89c4\u5212\u8c03\u6574 <code>ovn.kubernetes.io/ip_pool</code> \u4e2d IP \u7684\u6570\u91cf\u3002</li> </ol>"},{"location":"guide/static-ip-mac/#statefulset","title":"StatefulSet \u56fa\u5b9a\u5730\u5740","text":"<p>StatefulSet \u9ed8\u8ba4\u652f\u6301\u56fa\u5b9a IP\uff0c\u800c\u4e14\u548c\u5176\u4ed6 Workload \u76f8\u540c\uff0c\u53ef\u4ee5\u4f7f\u7528 <code>ovn.kubernetes.io/ip_pool</code> \u6765\u6307\u5b9a Pod \u4f7f\u7528\u7684 IP \u8303\u56f4\u3002</p> <p>\u7531\u4e8e StatefulSet \u591a\u7528\u4e8e\u6709\u72b6\u6001\u670d\u52a1\uff0c\u5bf9\u7f51\u7edc\u6807\u793a\u7684\u56fa\u5b9a\u6709\u66f4\u9ad8\u7684\u8981\u6c42\uff0cKube-OVN \u505a\u4e86\u7279\u6b8a\u7684\u5f3a\u5316\uff1a</p> <ol> <li>Pod \u4f1a\u6309\u987a\u5e8f\u5206\u914d <code>ovn.kubernetes.io/ip_pool</code> \u4e2d\u7684 IP\u3002\u4f8b\u5982 StatefulSet \u7684\u540d\u5b57\u4e3a web\uff0c\u5219 web-0 \u4f1a\u4f7f\u7528 <code>ovn.kubernetes.io/ip_pool</code> \u4e2d\u7684\u7b2c\u4e00\u4e2a IP\uff0c web-1 \u4f1a\u4f7f\u7528\u7b2c\u4e8c\u4e2a IP\uff0c\u4ee5\u6b64\u7c7b\u63a8\u3002</li> <li>StatefulSet Pod \u5728\u66f4\u65b0\u6216\u5220\u9664\u7684\u8fc7\u7a0b\u4e2d OVN \u4e2d\u7684 logical_switch_port \u4e0d\u4f1a\u5220\u9664\uff0c\u65b0\u751f\u6210\u7684 Pod \u76f4\u63a5\u590d\u7528\u65e7\u7684 interface \u4fe1\u606f\u3002\u56e0\u6b64 Pod \u53ef\u4ee5\u590d\u7528 IP/Mac \u53ca\u5176\u4ed6\u7f51\u7edc\u4fe1\u606f\uff0c\u8fbe\u5230\u548c StatefulSet Volume \u7c7b\u4f3c\u7684\u72b6\u6001\u4fdd\u7559\u529f\u80fd\u3002</li> <li>\u57fa\u4e8e 2 \u7684\u80fd\u529b\uff0c\u5bf9\u4e8e\u6ca1\u6709 <code>ovn.kubernetes.io/ip_pool</code> \u6ce8\u89e3\u7684 StatefulSet\uff0cPod \u7b2c\u4e00\u6b21\u751f\u6210\u65f6\u4f1a\u968f\u673a\u5206\u914d IP/Mac\uff0c\u4e4b\u540e\u5728\u6574\u4e2a StatefulSet \u7684\u751f\u547d\u5468\u671f\u5185\uff0c\u7f51\u7edc\u4fe1\u606f\u90fd\u4f1a\u4fdd\u6301\u56fa\u5b9a\u3002</li> </ol>"},{"location":"guide/static-ip-mac/#statefulset_1","title":"StatefulSet \u793a\u4f8b","text":"<pre><code>apiVersion: apps/v1\nkind: StatefulSet\nmetadata:\nname: web\nspec:\nserviceName: \"nginx\"\nreplicas: 2\nselector:\nmatchLabels:\napp: nginx\ntemplate:\nmetadata:\nlabels:\napp: nginx\nspec:\ncontainers:\n- name: nginx\nimage: docker.io/library/nginx:alpine\nports:\n- containerPort: 80\nname: web\n</code></pre> <p>\u53ef\u4ee5\u5c1d\u8bd5\u5220\u9664 StatefulSet \u4e0b Pod \u89c2\u5bdf Pod IP \u53d8\u5316\u4fe1\u606f\u3002</p>"},{"location":"guide/static-ip-mac/#kubevirt-vm","title":"KubeVirt VM \u56fa\u5b9a\u5730\u5740","text":"<p>\u9488\u5bf9 KubeVirt \u521b\u5efa\u7684 VM \u5b9e\u4f8b\uff0c<code>kube-ovn-controller</code> \u53ef\u4ee5\u6309\u7167\u7c7b\u4f3c StatefulSet Pod \u7684\u65b9\u5f0f\u8fdb\u884c IP \u5730\u5740\u5206\u914d\u548c\u7ba1\u7406\u3002 \u4ee5\u8fbe\u5230 VM \u5b9e\u4f8b\u5728\u751f\u547d\u5468\u671f\u5185\u542f\u505c\uff0c\u5347\u7ea7\uff0c\u8fc1\u79fb\u7b49\u64cd\u4f5c\u8fc7\u7a0b\u4e2d\u5730\u5740\u56fa\u5b9a\u4e0d\u53d8\uff0c\u66f4\u7b26\u865a\u62df\u5316\u5408\u7528\u6237\u7684\u5b9e\u9645\u4f7f\u7528\u4f53\u9a8c\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/subnet/","title":"\u5b50\u7f51\u4f7f\u7528","text":"<p>\u5b50\u7f51\u662f Kube-OVN \u4e2d\u7684\u4e00\u4e2a\u6838\u5fc3\u6982\u5ff5\u548c\u57fa\u672c\u4f7f\u7528\u5355\u5143\uff0cKube-OVN \u4f1a\u4ee5\u5b50\u7f51\u6765\u7ec4\u7ec7 IP \u548c\u7f51\u7edc\u914d\u7f6e\uff0c\u6bcf\u4e2a Namespace \u53ef\u4ee5\u5f52\u5c5e\u4e8e\u7279\u5b9a\u7684\u5b50\u7f51\uff0c Namespace \u4e0b\u7684 Pod \u4f1a\u81ea\u52a8\u4ece\u6240\u5c5e\u7684\u5b50\u7f51\u4e2d\u83b7\u53d6 IP \u5e76\u5171\u4eab\u5b50\u7f51\u7684\u7f51\u7edc\u914d\u7f6e\uff08CIDR\uff0c\u7f51\u5173\u7c7b\u578b\uff0c\u8bbf\u95ee\u63a7\u5236\uff0cNAT \u63a7\u5236\u7b49\uff09\u3002</p> <p>\u548c\u5176\u4ed6 CNI \u7684\u6bcf\u4e2a\u8282\u70b9\u7ed1\u5b9a\u4e00\u4e2a\u5b50\u7f51\u7684\u5b9e\u73b0\u4e0d\u540c\uff0c\u5728 Kube-OVN \u4e2d\u5b50\u7f51\u4e3a\u4e00\u4e2a\u5168\u5c40\u7684\u865a\u62df\u7f51\u7edc\u914d\u7f6e\uff0c\u540c\u4e00\u4e2a\u5b50\u7f51\u7684\u5730\u5740\u53ef\u4ee5\u5206\u5e03\u5728\u4efb\u610f\u4e00\u4e2a\u8282\u70b9\u4e0a\u3002</p> <p></p> <p>Overlay \u548c Underlay \u7684\u5b50\u7f51\u5728\u4f7f\u7528\u548c\u914d\u7f6e\u4e0a\u5b58\u5728\u4e00\u4e9b\u5dee\u5f02\uff0c\u672c\u6587\u6863\u5c06\u4f1a\u4ecb\u7ecd\u4e0d\u540c\u7c7b\u578b\u5b50\u7f51\u7684\u4e00\u4e9b\u5171\u540c\u914d\u7f6e\u548c\u5dee\u5f02\u5316\u529f\u80fd\u3002</p>"},{"location":"guide/subnet/#_2","title":"\u9ed8\u8ba4\u5b50\u7f51","text":"<p>\u4e3a\u4e86\u65b9\u4fbf\u7528\u6237\u7684\u5feb\u901f\u4e0a\u624b\u4f7f\u7528\uff0cKube-OVN \u5185\u7f6e\u4e86\u4e00\u4e2a\u9ed8\u8ba4\u5b50\u7f51\uff0c\u6240\u6709\u672a\u663e\u5f0f\u58f0\u660e\u5b50\u7f51\u5f52\u5c5e\u7684 Namespace \u4f1a\u81ea\u52a8\u4ece\u9ed8\u8ba4\u5b50\u7f51\u4e2d\u5206\u914d IP\uff0c \u5e76\u4f7f\u7528\u9ed8\u8ba4\u5b50\u7f51\u7684\u7f51\u7edc\u4fe1\u606f\u3002\u8be5\u5b50\u7f51\u7684\u914d\u7f6e\u4e3a\u5b89\u88c5\u65f6\u6307\u5b9a\uff0c\u53ef\u4ee5\u53c2\u8003\u5185\u7f6e\u7f51\u7edc\u8bbe\u7f6e\uff0c \u5982\u679c\u8981\u5728\u5b89\u88c5\u540e\u4fee\u6539\u9ed8\u8ba4\u7f51\u7edc\u7684 CIDR \u8bf7\u53c2\u8003\u4fee\u6539\u9ed8\u8ba4\u7f51\u7edc\u3002</p> <p>\u5728 Overlay \u6a21\u5f0f\u4e0b\uff0c\u9ed8\u8ba4\u5b50\u7f51\u4f7f\u7528\u4e86\u5206\u5e03\u5f0f\u7f51\u5173\u5e76\u5bf9\u51fa\u7f51\u6d41\u91cf\u8fdb\u884c NAT \u8f6c\u6362\uff0c\u5176\u884c\u4e3a\u548c Flannel \u7684\u9ed8\u8ba4\u884c\u4e3a\u57fa\u672c\u4e00\u81f4\uff0c \u7528\u6237\u65e0\u9700\u989d\u5916\u7684\u914d\u7f6e\u5373\u53ef\u4f7f\u7528\u5230\u5927\u90e8\u5206\u7684\u7f51\u7edc\u529f\u80fd\u3002</p> <p>\u5728 Underlay \u6a21\u5f0f\u4e0b\uff0c\u9ed8\u8ba4\u5b50\u7f51\u4f7f\u7528\u7269\u7406\u7f51\u5173\u4f5c\u4e3a\u51fa\u7f51\u7f51\u5173\uff0c\u5e76\u5f00\u542f arping \u68c0\u67e5\u7f51\u7edc\u8fde\u901a\u6027\u3002</p>"},{"location":"guide/subnet/#_3","title":"\u67e5\u770b\u9ed8\u8ba4\u5b50\u7f51","text":"<p>\u9ed8\u8ba4\u5b50\u7f51 spec \u4e2d\u7684 default \u5b57\u6bb5\u4e3a true\uff0c\u4e00\u4e2a\u96c6\u7fa4\u4e0b\u53ea\u6709\u4e00\u4e2a\u9ed8\u8ba4\u5b50\u7f51\uff0c\u9ed8\u8ba4\u540d\u4e3a <code>ovn-default</code>\u3002</p> <p>\u67e5\u770b\u9ed8\u8ba4\u5b50\u7f51\uff1a</p> <pre><code># kubectl get subnet ovn-default -o yaml\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n  creationTimestamp: \"2019-08-06T09:33:43Z\"\ngeneration: 1\nname: ovn-default\n  resourceVersion: \"1571334\"\nselfLink: /apis/kubeovn.io/v1/subnets/ovn-default\n  uid: 7e2451f8-fb44-4f7f-b3e0-cfd27f6fd5d6\nspec:\n  cidrBlock: 10.16.0.0/16\n  default: true\nexcludeIps:\n  - 10.16.0.1\n  gateway: 10.16.0.1\n  gatewayType: distributed\n  natOutgoing: true\nprivate: false\nprotocol: IPv4\n</code></pre>"},{"location":"guide/subnet/#join","title":"Join \u5b50\u7f51","text":"<p>\u5728 Kubernetes \u7684\u7f51\u7edc\u89c4\u8303\u4e2d\uff0c\u8981\u6c42 Node \u53ef\u4ee5\u548c\u6240\u6709\u7684 Pod \u76f4\u63a5\u901a\u4fe1\u3002 \u4e3a\u4e86\u5728 Overlay \u7f51\u7edc\u6a21\u5f0f\u4e0b\u8fbe\u5230\u8fd9\u4e2a\u76ee\u7684\uff0c Kube-OVN \u521b\u5efa\u4e86\u4e00\u4e2a <code>join</code> \u5b50\u7f51\uff0c \u5e76\u5728\u6bcf\u4e2a Node \u8282\u70b9\u521b\u5efa\u4e86\u4e00\u5757\u865a\u62df\u7f51\u5361 ovn0 \u63a5\u5165 <code>join</code> \u5b50\u7f51\uff0c\u901a\u8fc7\u8be5\u7f51\u7edc\u5b8c\u6210\u8282\u70b9\u548c Pod \u4e4b\u95f4\u7684\u7f51\u7edc\u4e92\u901a\u3002</p> <p>\u8be5\u5b50\u7f51\u7684\u914d\u7f6e\u4e3a\u5b89\u88c5\u65f6\u6307\u5b9a\uff0c\u53ef\u4ee5\u53c2\u8003\u5185\u7f6e\u7f51\u7edc\u8bbe\u7f6e\uff0c\u5982\u679c\u8981\u5728\u5b89\u88c5\u540e\u4fee\u6539\u3002 join \u5b50\u7f51\u7684 CIDR \u8bf7\u53c2\u8003\u4fee\u6539 Join \u5b50\u7f51</p>"},{"location":"guide/subnet/#join_1","title":"\u67e5\u770b Join \u5b50\u7f51","text":"<p>\u8be5\u5b50\u7f51\u9ed8\u8ba4\u540d\u4e3a <code>join</code> \u4e00\u822c\u65e0\u9700\u5bf9\u8be5\u5b50\u7f51 CIDR \u5916\u7684\u5176\u4ed6\u7f51\u7edc\u914d\u7f6e\u8fdb\u884c\u4fee\u6539\u3002</p> <pre><code># kubectl get subnet join -o yaml\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n  creationTimestamp: \"2019-08-06T09:33:43Z\"\ngeneration: 1\nname: join\n  resourceVersion: \"1571333\"\nselfLink: /apis/kubeovn.io/v1/subnets/join\n  uid: 9c744810-c678-4d50-8a7d-b8ec12ef91b8\nspec:\n  cidrBlock: 100.64.0.0/16\n  default: false\nexcludeIps:\n  - 100.64.0.1\n  gateway: 100.64.0.1\n  gatewayNode: \"\"\ngatewayType: \"\"\nnatOutgoing: false\nprivate: false\nprotocol: IPv4\n</code></pre> <p>\u5728 node \u8282\u70b9\u67e5\u770b ovn0 \u7f51\u5361\uff1a</p> <pre><code># ifconfig ovn0\novn0: flags=4163&lt;UP,BROADCAST,RUNNING,MULTICAST&gt;  mtu 1420\ninet 100.64.0.4  netmask 255.255.0.0  broadcast 100.64.255.255\n        inet6 fe80::800:ff:fe40:5  prefixlen 64  scopeid 0x20&lt;link&gt;\n        ether 0a:00:00:40:00:05  txqueuelen 1000  (Ethernet)\nRX packets 18  bytes 1428 (1.3 KiB)\nRX errors 0  dropped 0  overruns 0  frame 0\nTX packets 19  bytes 1810 (1.7 KiB)\nTX errors 0  dropped 0 overruns 0  carrier 0  collisions 0\n</code></pre>"},{"location":"guide/subnet/#_4","title":"\u521b\u5efa\u81ea\u5b9a\u4e49\u5b50\u7f51","text":"<p>\u8fd9\u91cc\u6211\u4eec\u4ecb\u7ecd\u521b\u5efa\u4e00\u4e2a\u5b50\u7f51\uff0c\u5e76\u5c06\u5176\u548c\u67d0\u4e2a Namespace \u505a\u5173\u8054\u7684\u57fa\u672c\u64cd\u4f5c\uff0c\u66f4\u591a\u9ad8\u7ea7\u914d\u7f6e\u8bf7\u53c2\u8003\u540e\u7eed\u5185\u5bb9\u3002</p>"},{"location":"guide/subnet/#_5","title":"\u521b\u5efa\u5b50\u7f51","text":"<pre><code>cat &lt;&lt;EOF | kubectl create -f -\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n  name: subnet1\nspec:\n  protocol: IPv4\n  cidrBlock: 10.66.0.0/16\n  excludeIps:\n  - 10.66.0.1..10.66.0.10\n  - 10.66.0.101..10.66.0.151\n  gateway: 10.66.0.1\n  gatewayType: distributed\n  natOutgoing: true\n  routeTable: \"\"\n  namespaces:\n  - ns1\n  - ns2\nEOF\n</code></pre> <ul> <li><code>cidrBlock</code>: \u5b50\u7f51 CIDR \u8303\u56f4\uff0c\u540c\u4e00\u4e2a VPC \u4e0b\u7684\u4e0d\u540c Subnet CIDR \u4e0d\u80fd\u91cd\u53e0\u3002</li> <li><code>excludeIps</code>: \u4fdd\u7559\u5730\u5740\u5217\u8868\uff0c\u5bb9\u5668\u7f51\u7edc\u5c06\u4e0d\u4f1a\u81ea\u52a8\u5206\u914d\u5217\u8868\u5185\u7684\u5730\u5740\uff0c\u53ef\u7528\u505a\u56fa\u5b9a IP \u5730\u5740\u5206\u914d\u6bb5\uff0c\u4e5f\u53ef\u5728 Underlay \u6a21\u5f0f\u4e0b\u907f\u514d\u548c\u7269\u7406\u7f51\u7edc\u4e2d\u5df2\u6709\u8bbe\u5907\u51b2\u7a81\u3002</li> <li><code>gateway</code>\uff1a\u8be5\u5b50\u7f51\u7f51\u5173\u5730\u5740\uff0cOverlay \u6a21\u5f0f\u4e0b Kube-OVN \u4f1a\u81ea\u52a8\u5206\u914d\u5bf9\u5e94\u7684\u903b\u8f91\u7f51\u5173\uff0cUnderlay \u6a21\u5f0f\u4e0b\u8be5\u5730\u5740\u9700\u4e3a\u5e95\u5c42\u7269\u7406\u7f51\u5173\u5730\u5740\u3002</li> <li><code>namespaces</code>: \u7ed1\u5b9a\u8be5\u5b50\u7f51\u7684 Namespace \u5217\u8868\uff0c\u7ed1\u5b9a\u540e Namespace \u4e0b\u7684 Pod \u5c06\u4f1a\u4ece\u5f53\u524d\u5b50\u7f51\u5206\u914d\u5730\u5740\u3002</li> <li><code>routeTable</code>: \u5173\u8054\u7684\u8def\u7531\u8868\uff0c\u9ed8\u8ba4\u5173\u8054\u4e3b\u8def\u7531\u8868\uff0c\u8def\u7531\u8868\u5b9a\u4e49\u8bf7\u53c2\u8003\u9759\u6001\u8def\u7531</li> </ul>"},{"location":"guide/subnet/#_6","title":"\u9a8c\u8bc1\u5b50\u7f51\u7ed1\u5b9a\u751f\u6548","text":"<pre><code># kubectl create ns ns1\nnamespace/ns1 created\n\n# kubectl run nginx --image=docker.io/library/nginx:alpine -n ns1\ndeployment.apps/nginx created\n\n# kubectl get pod -n ns1 -o wide\nNAME                     READY   STATUS    RESTARTS   AGE   IP           NODE    NOMINATED NODE   READINESS GATES\nnginx-74d5899f46-n8wtg   1/1     Running   0          10s   10.66.0.11   node1   &lt;none&gt;           &lt;none&gt;\n</code></pre>"},{"location":"guide/subnet/#overlay","title":"Overlay \u5b50\u7f51\u7f51\u5173\u914d\u7f6e","text":"<p>\u8be5\u529f\u80fd\u53ea\u5bf9 Overlay \u6a21\u5f0f\u5b50\u7f51\u751f\u6548\uff0cUnderlay \u7c7b\u578b\u5b50\u7f51\u8bbf\u95ee\u5916\u90e8\u7f51\u7edc\u9700\u8981\u501f\u52a9\u5e95\u5c42\u7269\u7406\u7f51\u5173\u3002</p> <p>Overlay \u5b50\u7f51\u4e0b\u7684 Pod \u9700\u8981\u901a\u8fc7\u7f51\u5173\u6765\u8bbf\u95ee\u96c6\u7fa4\u5916\u90e8\u7f51\u7edc\uff0cKube-OVN \u76ee\u524d\u652f\u6301\u4e24\u79cd\u7c7b\u578b\u7684\u7f51\u5173\uff1a \u5206\u5e03\u5f0f\u7f51\u5173\u548c\u96c6\u4e2d\u5f0f\u7f51\u5173\uff0c\u7528\u6237\u53ef\u4ee5\u5728\u5b50\u7f51\u4e2d\u5bf9\u7f51\u5173\u7684\u7c7b\u578b\u8fdb\u884c\u8c03\u6574\u3002</p> <p>\u4e24\u79cd\u7c7b\u578b\u7f51\u5173\u5747\u652f\u6301 <code>natOutgoing</code> \u8bbe\u7f6e\uff0c\u7528\u6237\u53ef\u4ee5\u9009\u62e9 Pod \u8bbf\u95ee\u5916\u7f51\u65f6\u662f\u5426\u9700\u8981\u8fdb\u884c snat\u3002</p>"},{"location":"guide/subnet/#_7","title":"\u5206\u5e03\u5f0f\u7f51\u5173","text":"<p>\u5b50\u7f51\u7684\u9ed8\u8ba4\u7c7b\u578b\u7f51\u5173\uff0c\u6bcf\u4e2a node \u4f1a\u4f5c\u4e3a\u5f53\u524d node \u4e0a pod \u8bbf\u95ee\u5916\u90e8\u7f51\u7edc\u7684\u7f51\u5173\u3002 \u6570\u636e\u5305\u4f1a\u901a\u8fc7\u672c\u673a\u7684 <code>ovn0</code> \u7f51\u5361\u6d41\u5165\u4e3b\u673a\u7f51\u7edc\u6808\uff0c\u518d\u6839\u636e\u4e3b\u673a\u7684\u8def\u7531\u89c4\u5219\u8fdb\u884c\u51fa\u7f51\u3002 \u5f53 <code>natOutgoing</code> \u4e3a <code>true</code> \u65f6\uff0cPod \u8bbf\u95ee\u5916\u90e8\u7f51\u7edc\u5c06\u4f1a\u4f7f\u7528\u5f53\u524d\u6240\u5728\u5bbf\u4e3b\u673a\u7684 IP\u3002</p> <p></p> <p>\u5b50\u7f51\u793a\u4f8b\uff0c\u5176\u4e2d <code>gatewayType</code> \u5b57\u6bb5\u4e3a <code>distributed</code>\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: distributed\nspec:\nprotocol: IPv4\ncidrBlock: 10.166.0.0/16\ndefault: false\nexcludeIps:\n- 10.166.0.1\ngateway: 10.166.0.1\ngatewayType: distributed\nnatOutgoing: true\n</code></pre>"},{"location":"guide/subnet/#_8","title":"\u96c6\u4e2d\u5f0f\u7f51\u5173","text":"<p>\u5982\u679c\u5e0c\u671b\u5b50\u7f51\u5185\u6d41\u91cf\u8bbf\u95ee\u5916\u7f51\u4f7f\u7528\u56fa\u5b9a\u7684 IP\uff0c\u4ee5\u4fbf\u5ba1\u8ba1\u548c\u767d\u540d\u5355\u7b49\u5b89\u5168\u64cd\u4f5c\uff0c\u53ef\u4ee5\u5728\u5b50\u7f51\u4e2d\u8bbe\u7f6e\u7f51\u5173\u7c7b\u578b\u4e3a\u96c6\u4e2d\u5f0f\u7f51\u5173\u3002 \u5728\u96c6\u4e2d\u5f0f\u7f51\u5173\u6a21\u5f0f\u4e0b\uff0cPod \u8bbf\u95ee\u5916\u7f51\u7684\u6570\u636e\u5305\u4f1a\u9996\u5148\u88ab\u8def\u7531\u5230\u7279\u5b9a\u8282\u70b9\u7684 <code>ovn0</code> \u7f51\u5361\uff0c\u518d\u901a\u8fc7\u4e3b\u673a\u7684\u8def\u7531\u89c4\u5219\u8fdb\u884c\u51fa\u7f51\u3002 \u5f53 <code>natOutgoing</code> \u4e3a <code>true</code> \u65f6\uff0cPod \u8bbf\u95ee\u5916\u90e8\u7f51\u7edc\u5c06\u4f1a\u4f7f\u7528\u7279\u5b9a\u5bbf\u4e3b\u673a\u7684 IP\u3002</p> <p>\u5b50\u7f51\u793a\u4f8b\uff0c\u5176\u4e2d <code>gatewayType</code> \u5b57\u6bb5\u4e3a <code>centralized</code>\uff0c<code>gatewayNode</code> \u4e3a\u7279\u5b9a\u673a\u5668\u5728 Kubernetes \u4e2d\u7684 NodeName\u3002 \u5176\u4e2d <code>gatewayNode</code> \u5b57\u6bb5\u53ef\u4ee5\u4e3a\u9017\u53f7\u5206\u9694\u7684\u591a\u53f0\u4e3b\u673a\u3002</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: centralized\nspec:\nprotocol: IPv4\ncidrBlock: 10.166.0.0/16\ndefault: false\nexcludeIps:\n- 10.166.0.1\ngateway: 10.166.0.1\ngatewayType: centralized\ngatewayNode: \"node1,node2\"\nnatOutgoing: true\n</code></pre> <ul> <li>\u96c6\u4e2d\u5f0f\u7f51\u5173\u5982\u679c\u5e0c\u671b\u6307\u5b9a\u673a\u5668\u7684\u7279\u5b9a\u7f51\u5361\u8fdb\u884c\u51fa\u7f51\uff0c<code>gatewayNode</code> \u53ef\u66f4\u6539\u4e3a <code>kube-ovn-worker:172.18.0.2, kube-ovn-control-plane:172.18.0.3</code> \u683c\u5f0f\u3002</li> <li>\u96c6\u4e2d\u5f0f\u7f51\u5173\u9ed8\u8ba4\u4e3a\u4e3b\u5907\u6a21\u5f0f\uff0c\u53ea\u6709\u4e3b\u8282\u70b9\u8fdb\u884c\u6d41\u91cf\u8f6c\u53d1\uff0c \u5982\u679c\u9700\u8981\u5207\u6362\u4e3a ECMP \u6a21\u5f0f\uff0c\u8bf7\u53c2\u8003\u96c6\u4e2d\u5f0f\u7f51\u5173 ECMP \u5f00\u542f\u8bbe\u7f6e\u3002</li> <li>\u4ece Kube-OVN v1.12.0 \u7248\u672c\u5f00\u59cb\uff0c\u5728 subnet crd \u5b9a\u4e49\u4e2d\u589e\u52a0\u4e86 spec \u5b57\u6bb5 <code>enableEcmp</code>\uff0c\u5c06\u96c6\u4e2d\u5f0f\u5b50\u7f51 ECMP \u5f00\u5173\u63a7\u5236\u8fc1\u79fb\u5230\u5b50\u7f51\u5c42\u7ea7\uff0c\u53ef\u4ee5\u57fa\u4e8e\u4e0d\u540c\u7684\u5b50\u7f51\u5206\u522b\u8bbe\u7f6e\u662f\u5426\u5f00\u542f ECMP \u6a21\u5f0f\u3002\u539f\u6709\u7684 <code>kube-ovn-controller</code> Deployment \u4e2d\u7684 <code>enable-ecmp</code> \u53c2\u6570\u4e0d\u518d\u4f7f\u7528\u3002\u4e4b\u524d\u7248\u672c\u5347\u7ea7\u5230 v1.12.0 \u4e4b\u540e\uff0c\u5b50\u7f51\u5f00\u5173\u4f1a\u81ea\u52a8\u7ee7\u627f\u539f\u6709\u7684\u5168\u5c40\u5f00\u5173\u53c2\u6570\u53d6\u503c\u3002</li> </ul>"},{"location":"guide/subnet/#acl","title":"\u5b50\u7f51 ACL \u8bbe\u7f6e","text":"<p>\u5bf9\u4e8e\u6709\u7ec6\u7c92\u5ea6 ACL \u63a7\u5236\u7684\u573a\u666f\uff0cKube-OVN \u7684 Subnet \u63d0\u4f9b\u4e86 ACL \u89c4\u5219\u7684\u8bbe\u7f6e\uff0c\u53ef\u4ee5\u5b9e\u73b0\u7f51\u7edc\u89c4\u5219\u7684\u7cbe\u7ec6\u63a7\u5236\u3002</p> <p>Subnet \u4e2d\u7684 ACL \u89c4\u5219\u548c OVN \u7684 ACL \u89c4\u5219\u4e00\u81f4\uff0c\u76f8\u5173\u5b57\u6bb5\u5185\u5bb9\u53ef\u4ee5\u53c2\u8003 ovn-nb ACL Table\uff0c <code>match</code> \u5b57\u6bb5\u652f\u6301\u7684\u5b57\u6bb5\u53ef\u53c2\u8003 ovn-sb Logical Flow Table\u3002</p> <p>\u5141\u8bb8 IP \u5730\u5740\u4e3a <code>10.10.0.2</code> \u7684 Pod \u8bbf\u95ee\u6240\u6709\u5730\u5740\uff0c\u4f46\u4e0d\u5141\u8bb8\u5176\u4ed6\u5730\u5740\u4e3b\u52a8\u8bbf\u95ee\u81ea\u5df1\u7684 ACL \u89c4\u5219\u793a\u4f8b\u5982\u4e0b\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: acl\nspec:\nacls:\n- action: drop\ndirection: to-lport\nmatch: ip4.dst == 10.10.0.2 &amp;&amp; ip\npriority: 1002\n- action: allow-related\ndirection: from-lport\nmatch: ip4.src == 10.10.0.2 &amp;&amp; ip\npriority: 1002\ncidrBlock: 10.10.0.0/24\n</code></pre>"},{"location":"guide/subnet/#_9","title":"\u5b50\u7f51\u9694\u79bb\u8bbe\u7f6e","text":"<p>\u5b50\u7f51 ACL \u7684\u529f\u80fd\u53ef\u4ee5\u8986\u76d6\u5b50\u7f51\u9694\u79bb\u7684\u529f\u80fd\uff0c\u5e76\u6709\u66f4\u597d\u7684\u7075\u6d3b\u6027\uff0c\u6211\u4eec\u63a8\u8350\u4f7f\u7528\u5b50\u7f51 ACL \u6765\u505a\u76f8\u5e94\u7684\u914d\u7f6e\u3002</p> <p>\u9ed8\u8ba4\u60c5\u51b5\u4e0b Kube-OVN \u521b\u5efa\u7684\u5b50\u7f51\u4e4b\u95f4\u53ef\u4ee5\u76f8\u4e92\u901a\u4fe1\uff0cPod \u4e5f\u53ef\u4ee5\u901a\u8fc7\u7f51\u5173\u8bbf\u95ee\u5916\u90e8\u7f51\u7edc\u3002</p> <p>\u5982\u9700\u5bf9\u5b50\u7f51\u95f4\u7684\u8bbf\u95ee\u8fdb\u884c\u63a7\u5236\uff0c\u53ef\u4ee5\u5728\u5b50\u7f51 CRD \u4e2d\u5c06 <code>private</code> \u8bbe\u7f6e\u4e3a true\uff0c\u5219\u8be5\u5b50\u7f51\u5c06\u548c\u5176\u4ed6\u5b50\u7f51\u4ee5\u53ca\u5916\u90e8\u7f51\u7edc\u9694\u79bb\uff0c \u53ea\u80fd\u8fdb\u884c\u5b50\u7f51\u5185\u90e8\u7684\u901a\u4fe1\u3002\u5982\u9700\u5f00\u767d\u540d\u5355\uff0c\u53ef\u4ee5\u901a\u8fc7 <code>allowSubnets</code> \u8fdb\u884c\u8bbe\u7f6e\u3002<code>allowSubnets</code> \u5185\u7684\u7f51\u6bb5\u548c\u8be5\u5b50\u7f51\u53ef\u4ee5\u53cc\u5411\u4e92\u8bbf\u3002</p>"},{"location":"guide/subnet/#_10","title":"\u5f00\u542f\u8bbf\u95ee\u63a7\u5236\u7684\u5b50\u7f51\u793a\u4f8b","text":"<pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: private\nspec:\nprotocol: IPv4\ndefault: false\nnamespaces:\n- ns1\n- ns2\ncidrBlock: 10.69.0.0/16\nprivate: true\nallowSubnets:\n- 10.16.0.0/16\n- 10.18.0.0/16\n</code></pre>"},{"location":"guide/subnet/#underlay","title":"Underlay \u76f8\u5173\u9009\u9879","text":"<p>\u8be5\u90e8\u5206\u529f\u80fd\u53ea\u5bf9 Underlay \u7c7b\u578b\u5b50\u7f51\u751f\u6548\u3002</p> <ul> <li><code>vlan</code>: \u5982\u679c\u4f7f\u7528 Underlay \u7f51\u7edc\uff0c\u8be5\u5b57\u6bb5\u7528\u6765\u63a7\u5236\u8be5 Subnet \u548c\u54ea\u4e2a Vlan CR \u8fdb\u884c\u7ed1\u5b9a\u3002\u8be5\u9009\u9879\u9ed8\u8ba4\u4e3a\u7a7a\u5b57\u7b26\u4e32\uff0c\u5373\u4e0d\u4f7f\u7528 Underlay \u7f51\u7edc\u3002</li> <li><code>logicalGateway</code>: \u4e00\u4e9b Underlay \u73af\u5883\u4e3a\u7eaf\u4e8c\u5c42\u7f51\u7edc\uff0c\u4e0d\u5b58\u5728\u7269\u7406\u7684\u4e09\u5c42\u7f51\u5173\u3002\u5728\u8fd9\u79cd\u60c5\u51b5\u4e0b\u53ef\u4ee5\u501f\u52a9 OVN \u672c\u8eab\u7684\u80fd\u529b\u8bbe\u7f6e\u4e00\u4e2a\u865a\u62df\u7f51\u5173\uff0c\u5c06 Underlay   \u548c Overlay \u7f51\u7edc\u6253\u901a\u3002\u9ed8\u8ba4\u503c\u4e3a\uff1a<code>false</code>\u3002</li> </ul>"},{"location":"guide/subnet/#_11","title":"\u7f51\u5173\u68c0\u67e5\u8bbe\u7f6e","text":"<p>\u9ed8\u8ba4\u60c5\u51b5\u4e0b <code>kube-ovn-cni</code> \u5728\u542f\u52a8 Pod \u540e\u4f1a\u4f7f\u7528 ICMP \u6216 ARP \u534f\u8bae\u8bf7\u6c42\u7f51\u5173\u5e76\u7b49\u5f85\u8fd4\u56de\uff0c \u4ee5\u9a8c\u8bc1\u7f51\u7edc\u5de5\u4f5c\u6b63\u5e38\uff0c\u5728\u90e8\u5206 Underlay \u73af\u5883\u7f51\u5173\u65e0\u6cd5\u54cd\u5e94 ARP \u8bf7\u6c42\uff0c\u6216\u65e0\u9700\u7f51\u7edc\u5916\u90e8\u8054\u901a\u7684\u573a\u666f \u53ef\u4ee5\u5173\u95ed\u7f51\u5173\u68c0\u67e5\u3002</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: disable-gw-check\nspec:\ndisableGatewayCheck: true\n</code></pre>"},{"location":"guide/subnet/#multicast-snoop","title":"Multicast-Snoop \u914d\u7f6e","text":"<p>\u9ed8\u8ba4\u60c5\u51b5\u4e0b subnet \u4e0b\u7684 Pod \u5982\u679c\u53d1\u9001\u7ec4\u64ad\u62a5\u6587\uff0cOVN \u7684\u9ed8\u8ba4\u884c\u4e3a\u662f\u4f1a\u5e7f\u64ad\u7ec4\u64ad\u62a5\u6587\u5230\u5b50\u7f51\u4e0b\u6240\u6709\u7684 Pod\u3002\u5982\u679c\u5f00\u542f subnet \u7684 multicast snoop \u5f00\u5173\uff0cOVN \u4f1a\u6839\u636e <code>South Database</code> \u4e2d\u7684\u7ec4\u64ad\u8868 <code>Multicast_Group</code> \u6765\u8fdb\u884c\u8f6c\u53d1\uff0c\u800c\u4e0d\u5728\u8fdb\u884c\u5e7f\u64ad\u3002</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: sample1\nspec:\nenableMulticastSnoop: true\n</code></pre>"},{"location":"guide/subnet/#_12","title":"\u5176\u4ed6\u9ad8\u7ea7\u8bbe\u7f6e","text":"<ul> <li>IP \u6c60\u4f7f\u7528</li> <li>\u9ed8\u8ba4 VPC NAT \u7b56\u7565\u89c4\u5219</li> <li>QoS \u8bbe\u7f6e</li> <li>\u591a\u7f51\u5361\u7ba1\u7406</li> <li>DHCP \u9009\u9879</li> <li>\u5916\u90e8\u7f51\u5173\u8bbe\u7f6e</li> <li>\u96c6\u7fa4\u4e92\u8054\u8bbe\u7f6e</li> <li>\u865a\u62df IP \u8bbe\u7f6e</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/vpc-qos/","title":"VPC QoS","text":"<p>Kube-OVN \u652f\u6301\u4f7f\u7528 QoSPolicy CRD \u5bf9\u81ea\u5b9a\u4e49 VPC \u7684\u6d41\u91cf\u901f\u7387\u8fdb\u884c\u9650\u5236\u3002</p>"},{"location":"guide/vpc-qos/#eip-qos","title":"EIP QoS","text":"<p>\u5bf9 EIP \u8fdb\u884c\u9650\u901f\uff0c\u9650\u901f\u503c\u4e3a 1Mbps\uff0c\u4f18\u5148\u7ea7\u4e3a 1\uff0c\u8fd9\u91cc <code>shared=false</code>\uff0c\u8868\u793a\u8fd9\u4e2a QoSPolicy \u53ea\u80fd\u7ed9\u8fd9\u4e2a EIP \u4f7f\u7528\u4e14\u652f\u6301\u52a8\u6001\u4fee\u6539 QoSPolicy \u53bb\u53d8\u66f4 QoS \u89c4\u5219\u3002</p> <p>QoSPolicy \u914d\u7f6e\u5982\u4e0b\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: QoSPolicy\nmetadata:\nname: qos-eip-example\nspec:\nshared: false\nbindingType: EIP\nbandwidthLimitRules:\n- name: eip-ingress\nrateMax: \"1\" # Mbps\nburstMax: \"1\" # Mbps\npriority: 1\ndirection: ingress\n- name: eip-egress\nrateMax: \"1\" # Mbps\nburstMax: \"1\" # Mbps\npriority: 1\ndirection: egress\n</code></pre> <p>IptablesEIP \u914d\u7f6e\u5982\u4e0b\uff1a</p> <pre><code>kind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-1\nspec:\nnatGwDp: gw1\nqosPolicy: qos-eip-example\n</code></pre> <p><code>.spec.qosPolicy</code>\u7684\u503c\u652f\u6301\u521b\u5efa\u65f6\u4f20\u5165\uff0c\u4e5f\u652f\u6301\u521b\u5efa\u540e\u4fee\u6539\u3002</p>"},{"location":"guide/vpc-qos/#qos-eip","title":"\u67e5\u770b\u5df2\u542f\u7528 QoS \u7684 EIP","text":"<p>\u901a\u8fc7<code>label</code>\u67e5\u770b\u5df2\u7ecf\u8bbe\u7f6e\u5bf9\u5e94 qos \u7684 eip\uff1a</p> <pre><code># kubectl get eip  -l ovn.kubernetes.io/qos=qos-eip-example\nNAME    IP             MAC                 NAT   NATGWDP   READY\neip-1   172.18.11.24   00:00:00:34:41:0B   fip   gw1       true\n</code></pre>"},{"location":"guide/vpc-qos/#vpc-natgw-net1-qos","title":"VPC NATGW net1 \u7f51\u5361 QoS","text":"<p>\u5bf9 VPC NATGW \u7684 net1 \u7f51\u5361\u901f\u7387\u8fdb\u884c\u9650\u5236\uff0c\u9650\u901f\u503c\u4e3a 10Mbps\uff0c\u4f18\u5148\u7ea7\u4e3a 3\uff0c\u8fd9\u91cc <code>shared=true</code>\uff0c\u8868\u793a\u8fd9\u4e2a QoSPolicy \u53ef\u4ee5\u540c\u65f6\u7ed9\u591a\u4e2a\u8d44\u6e90\u4f7f\u7528\uff0c\u8fd9\u79cd\u573a\u666f\u4e0b\u4e0d\u5141\u8bb8\u4fee\u6539 QoSPolicy \u7684\u5185\u5bb9\u3002</p> <p>QoSPolicy \u914d\u7f6e\u5982\u4e0b\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: QoSPolicy\nmetadata:\nname: qos-natgw-example\nspec:\nshared: true\nbindingType: NATGW\nbandwidthLimitRules:\n- name: net1-ingress\ninterface: net1\nrateMax: \"10\" # Mbps\nburstMax: \"10\" # Mbps\npriority: 3\ndirection: ingress\n- name: net1-egress\ninterface: net1\nrateMax: \"10\" # Mbps\nburstMax: \"10\" # Mbps\npriority: 3\ndirection: egress\n</code></pre> <p>VpcNatGateway \u914d\u7f6e\u5982\u4e0b\uff1a</p> <pre><code>kind: VpcNatGateway\napiVersion: kubeovn.io/v1\nmetadata:\nname: gw1\nspec:\nvpc: test-vpc-1\nsubnet: net1\nlanIp: 10.0.1.254\nqosPolicy: qos-natgw-example\nselector:\n- \"kubernetes.io/hostname: kube-ovn-worker\"\n- \"kubernetes.io/os: linux\"\n</code></pre> <p><code>.spec.qosPolicy</code>\u7684\u503c\u652f\u6301\u521b\u5efa\u4f20\u5165\uff0c\u4e5f\u652f\u6301\u540e\u7eed\u4fee\u6539\u3002</p>"},{"location":"guide/vpc-qos/#net1-qos","title":"net1 \u7f51\u5361\u7279\u5b9a\u6d41\u91cf QoS","text":"<p>\u5bf9 net1 \u7f51\u5361\u4e0a\u7279\u5b9a\u6d41\u91cf\u8fdb\u884c\u9650\u901f\uff0c\u9650\u901f\u503c\u4e3a 5Mbps\uff0c\u4f18\u5148\u7ea7\u4e3a 2\uff0c\u8fd9\u91cc <code>shared=true</code>\uff0c\u8868\u793a\u8fd9\u4e2a QoSPolicy  \u53ef\u4ee5\u540c\u65f6\u7ed9\u591a\u4e2a\u8d44\u6e90\u4f7f\u7528\uff0c\u6b64\u65f6\u4e0d\u5141\u8bb8\u4fee\u6539 QoSPolicy \u7684\u5185\u5bb9\u3002</p> <p>QoSPolicy \u914d\u7f6e\u5982\u4e0b\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: QoSPolicy\nmetadata:\nname: qos-natgw-example\nspec:\nshared: true\nbindingType: NATGW\nbandwidthLimitRules:\n- name: net1-extip-ingress\ninterface: net1\nrateMax: \"5\" # Mbps\nburstMax: \"5\" # Mbps\npriority: 2\ndirection: ingress\nmatchType: ip\nmatchValue: src 172.18.11.22/32\n- name: net1-extip-egress\ninterface: net1\nrateMax: \"5\" # Mbps\nburstMax: \"5\" # Mbps\npriority: 2\ndirection: egress\nmatchType: ip\nmatchValue: dst 172.18.11.23/32\n</code></pre> <p>VpcNatGateway \u914d\u7f6e\u5982\u4e0b\uff1a</p> <pre><code>kind: VpcNatGateway\napiVersion: kubeovn.io/v1\nmetadata:\nname: gw1\nspec:\nvpc: test-vpc-1\nsubnet: net1\nlanIp: 10.0.1.254\nqosPolicy: qos-natgw-example\nselector:\n- \"kubernetes.io/hostname: kube-ovn-worker\"\n- \"kubernetes.io/os: linux\"\n</code></pre>"},{"location":"guide/vpc-qos/#qos-natgw","title":"\u67e5\u770b\u5df2\u542f\u7528 QoS \u7684 NATGW","text":"<p>\u901a\u8fc7<code>label</code>\u67e5\u770b\u5df2\u7ecf\u8bbe\u7f6e\u5bf9\u5e94 qos \u7684 eip\uff1a</p> <pre><code># kubectl get vpc-nat-gw  -l ovn.kubernetes.io/qos=qos-natgw-example\nNAME   VPC          SUBNET   LANIP\ngw1    test-vpc-1   net1     10.0.1.254\n</code></pre>"},{"location":"guide/vpc-qos/#qos","title":"\u67e5\u770b qos \u89c4\u5219","text":"<pre><code># kubectl get qos -A\nNAME                SHARED   BINDINGTYPE\nqos-eip-example     false    EIP\nqos-natgw-example   true     NATGW\n</code></pre>"},{"location":"guide/vpc-qos/#_1","title":"\u9650\u5236","text":"<ul> <li>\u53ea\u6709\u5728\u672a\u4f7f\u7528\u65f6\u624d\u80fd\u5220\u9664 QoS \u7b56\u7565\u3002\u56e0\u6b64\uff0c\u5728\u5220\u9664 QoS \u7b56\u7565\u4e4b\u524d\uff0c\u8bf7\u5148\u67e5\u770b\u5df2\u542f\u7528 QoS \u7684 EIP \u548c NATGW\uff0c\u53bb\u6389\u5b83\u4eec\u7684 <code>spec.qosPolicy</code> \u914d\u7f6e\u3002</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/vpc/","title":"VPC \u4f7f\u7528","text":"<p>Kube-OVN \u652f\u6301\u591a\u79df\u6237\u9694\u79bb\u7ea7\u522b\u7684 VPC \u7f51\u7edc\u3002\u4e0d\u540c VPC \u7f51\u7edc\u76f8\u4e92\u72ec\u7acb\uff0c\u53ef\u4ee5\u5206\u522b\u914d\u7f6e Subnet \u7f51\u6bb5\uff0c \u8def\u7531\u7b56\u7565\uff0c\u5b89\u5168\u7b56\u7565\uff0c\u51fa\u7f51\u7f51\u5173\uff0cEIP \u7b49\u914d\u7f6e\u3002</p> <p>VPC \u4e3b\u8981\u7528\u4e8e\u6709\u591a\u79df\u6237\u7f51\u7edc\u5f3a\u9694\u79bb\u7684\u573a\u666f\uff0c\u90e8\u5206 Kubernetes \u7f51\u7edc\u529f\u80fd\u5728\u591a\u79df\u6237\u7f51\u7edc\u4e0b\u5b58\u5728\u51b2\u7a81\u3002 \u4f8b\u5982\u8282\u70b9\u548c Pod \u4e92\u8bbf\uff0cNodePort \u529f\u80fd\uff0c\u57fa\u4e8e\u7f51\u7edc\u8bbf\u95ee\u7684\u5065\u5eb7\u68c0\u67e5\u548c DNS \u80fd\u529b\u5728\u591a\u79df\u6237\u7f51\u7edc\u573a\u666f\u6682\u4e0d\u652f\u6301\u3002 \u4e3a\u4e86\u65b9\u4fbf\u5e38\u89c1 Kubernetes \u7684\u4f7f\u7528\u573a\u666f\uff0cKube-OVN \u9ed8\u8ba4 VPC \u505a\u4e86\u7279\u6b8a\u8bbe\u8ba1\uff0c\u8be5 VPC \u4e0b\u7684 Subnet \u53ef\u4ee5\u6ee1\u8db3 Kubernetes \u89c4\u8303\u3002\u7528\u6237\u81ea\u5b9a\u4e49 VPC \u652f\u6301\u672c\u6587\u6863\u4ecb\u7ecd\u7684\u9759\u6001\u8def\u7531\uff0cEIP \u548c NAT \u7f51\u5173\u7b49\u529f\u80fd\u3002 \u5e38\u89c1\u9694\u79bb\u9700\u6c42\u53ef\u901a\u8fc7\u9ed8\u8ba4 VPC \u4e0b\u7684\u7f51\u7edc\u7b56\u7565\u548c\u5b50\u7f51 ACL \u5b9e\u73b0\uff0c\u5728\u4f7f\u7528\u81ea\u5b9a\u4e49 VPC \u524d\u8bf7\u660e\u786e\u662f\u5426\u9700\u8981 VPC \u7ea7\u522b\u7684\u9694\u79bb\uff0c\u5e76\u4e86\u89e3\u81ea\u5b9a\u4e49 VPC \u4e0b\u7684\u9650\u5236\u3002 \u5728 Underlay \u7f51\u7edc\u4e0b\uff0c\u7269\u7406\u4ea4\u6362\u673a\u8d1f\u8d23\u6570\u636e\u9762\u8f6c\u53d1\uff0cVPC \u65e0\u6cd5\u5bf9 Underlay \u5b50\u7f51\u8fdb\u884c\u9694\u79bb\u3002</p> <p></p>"},{"location":"guide/vpc/#vpc_1","title":"\u521b\u5efa\u81ea\u5b9a\u4e49 VPC","text":"<p>\u521b\u5efa\u4e24\u4e2a VPC\uff1a</p> <pre><code>kind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: test-vpc-1\nspec:\nnamespaces:\n- ns1\n---\nkind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: test-vpc-2\nspec:\nnamespaces:\n- ns2\n</code></pre> <ul> <li><code>namespaces</code> \u53ef\u4ee5\u9650\u5b9a\u53ea\u6709\u54ea\u4e9b Namespace \u53ef\u4ee5\u4f7f\u7528\u5f53\u524d VPC\uff0c\u82e5\u4e3a\u7a7a\u5219\u4e0d\u9650\u5b9a\u3002</li> </ul> <p>\u521b\u5efa\u4e24\u4e2a\u5b50\u7f51\uff0c\u5206\u5c5e\u4e24\u4e2a\u4e0d\u540c\u7684 VPC \u5e76\u6709\u76f8\u540c\u7684 CIDR:</p> <pre><code>kind: Subnet\napiVersion: kubeovn.io/v1\nmetadata:\nname: net1\nspec:\nvpc: test-vpc-1\ncidrBlock: 10.0.1.0/24\nprotocol: IPv4\nnamespaces:\n- ns1\n---\nkind: Subnet\napiVersion: kubeovn.io/v1\nmetadata:\nname: net2\nspec:\nvpc: test-vpc-2\ncidrBlock: 10.0.1.0/24\nprotocol: IPv4\nnamespaces:\n- ns2\n</code></pre> <p>\u5206\u522b\u5728\u4e24\u4e2a Namespace \u4e0b\u521b\u5efa Pod:</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nannotations:\novn.kubernetes.io/logical_switch: net1\nnamespace: ns1\nname: vpc1-pod\nspec:\ncontainers:\n- name: vpc1-pod\nimage: docker.io/library/nginx:alpine\n---\napiVersion: v1\nkind: Pod\nmetadata:\nannotations:\novn.kubernetes.io/logical_switch: net2\nnamespace: ns2\nname: vpc2-pod\nspec:\ncontainers:\n- name: vpc2-pod\nimage: docker.io/library/nginx:alpine\n</code></pre> <p>\u8fd0\u884c\u6210\u529f\u540e\u53ef\u89c2\u5bdf\u4e24\u4e2a Pod \u5730\u5740\u5c5e\u4e8e\u540c\u4e00\u4e2a CIDR\uff0c\u4f46\u7531\u4e8e\u8fd0\u884c\u5728\u4e0d\u540c\u7684\u79df\u6237 VPC\uff0c\u4e24\u4e2a Pod \u65e0\u6cd5\u76f8\u4e92\u8bbf\u95ee\u3002</p>"},{"location":"guide/vpc/#vpc-pod-livenessprobe-readinessprobe","title":"\u81ea\u5b9a\u4e49 VPC Pod \u652f\u6301 livenessProbe \u548c readinessProbe","text":"<p>\u7531\u4e8e\u5e38\u89c4\u914d\u7f6e\u4e0b\u81ea\u5b9a\u4e49 VPC \u4e0b\u7684 Pod \u548c\u8282\u70b9\u7684\u7f51\u7edc\u4e4b\u95f4\u5e76\u4e0d\u4e92\u901a\uff0c\u6240\u4ee5 kubelet \u53d1\u9001\u7684\u63a2\u6d4b\u62a5\u6587\u65e0\u6cd5\u5230\u8fbe\u81ea\u5b9a VPC \u5185\u7684 Pod\u3002Kube-OVN \u901a\u8fc7 TProxy \u5c06 kubelet \u53d1\u9001\u7684\u63a2\u6d4b\u62a5\u6587\u91cd\u5b9a\u5411\u5230\u81ea\u5b9a\u4e49 VPC \u5185\u7684 Pod\uff0c\u4ece\u800c\u5b9e\u73b0\u8fd9\u4e00\u529f\u80fd\u3002</p> <p>\u914d\u7f6e\u65b9\u6cd5\u5982\u4e0b\uff0c\u5728 Daemonset <code>kube-ovn-cni</code> \u4e2d\u589e\u52a0\u53c2\u6570 <code>--enable-tproxy=true</code>\uff1a</p> <pre><code>spec:\ntemplate:\nspec:\ncontainers:\n- args:\n- --enable-tproxy=true\n</code></pre> <p>\u8be5\u529f\u80fd\u9650\u5236\u6761\u4ef6\uff1a</p> <ol> <li>\u5f53\u540c\u4e00\u4e2a\u8282\u70b9\u4e0b\u51fa\u73b0\u4e0d\u540c VPC \u4e0b\u7684 Pod \u5177\u6709\u76f8\u540c\u7684 IP\uff0c\u63a2\u6d4b\u529f\u80fd\u5931\u6548\u3002</li> <li>\u76ee\u524d\u6682\u65f6\u53ea\u652f\u6301 <code>tcpSocket</code> \u548c <code>httpGet</code> \u4e24\u79cd\u63a2\u6d4b\u65b9\u5f0f\u3002</li> </ol>"},{"location":"guide/vpc/#vpc_2","title":"\u521b\u5efa VPC \u7f51\u5173","text":"<p>\u81ea\u5b9a\u4e49 VPC \u4e0b\u7684\u5b50\u7f51\u4e0d\u652f\u6301\u9ed8\u8ba4 VPC \u4e0b\u7684\u5206\u5e03\u5f0f\u7f51\u5173\u548c\u96c6\u4e2d\u5f0f\u7f51\u5173\u3002</p> <p>VPC \u5185\u5bb9\u5668\u8bbf\u95ee\u5916\u90e8\u7f51\u7edc\u9700\u8981\u901a\u8fc7 VPC \u7f51\u5173\uff0cVPC \u7f51\u5173\u53ef\u4ee5\u6253\u901a\u7269\u7406\u7f51\u7edc\u548c\u79df\u6237\u7f51\u7edc\uff0c\u5e76\u63d0\u4f9b \u6d6e\u52a8 IP\uff0cSNAT \u548c DNAT \u529f\u80fd\u3002</p> <p>VPC \u7f51\u5173\u529f\u80fd\u4f9d\u8d56 Multus-CNI \u7684\u591a\u7f51\u5361\u529f\u80fd\uff0c\u5b89\u88c5\u8bf7\u53c2\u8003 multus-cni\u3002</p>"},{"location":"guide/vpc/#_1","title":"\u914d\u7f6e\u5916\u90e8\u7f51\u7edc","text":"<pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: ovn-vpc-external-network\nspec:\nprotocol: IPv4\nprovider: ovn-vpc-external-network.kube-system\ncidrBlock: 192.168.0.0/24\ngateway: 192.168.0.1  # IP address of the physical gateway\nexcludeIps:\n- 192.168.0.1..192.168.0.10\n---\napiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: ovn-vpc-external-network\nnamespace: kube-system\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.0\",\n\"type\": \"macvlan\",\n\"master\": \"eth1\",\n\"mode\": \"bridge\",\n\"ipam\": {\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"ovn-vpc-external-network.kube-system\"\n}\n}'\n</code></pre> <ul> <li>\u8be5 Subnet \u7528\u6765\u7ba1\u7406\u53ef\u7528\u7684\u5916\u90e8\u5730\u5740\uff0c\u7f51\u6bb5\u5185\u7684\u5730\u5740\u5c06\u4f1a\u901a\u8fc7 Macvlan \u5206\u914d\u7ed9 VPC \u7f51\u5173\uff0c\u8bf7\u548c\u7f51\u7edc\u7ba1\u7406\u6c9f\u901a\u7ed9\u51fa\u53ef\u7528\u7684\u7269\u7406\u6bb5 IP\u3002</li> <li>VPC \u7f51\u5173\u4f7f\u7528 Macvlan \u505a\u7269\u7406\u7f51\u7edc\u914d\u7f6e\uff0c<code>NetworkAttachmentDefinition</code> \u7684 <code>master</code> \u9700\u4e3a\u5bf9\u5e94\u7269\u7406\u7f51\u8def\u7f51\u5361\u7684\u7f51\u5361\u540d\u3002</li> <li><code>name</code> \u5916\u90e8\u7f51\u7edc\u540d\u79f0\u3002</li> </ul> <p>\u5728 Macvlan \u6a21\u5f0f\u4e0b\uff0c\u9644\u5c5e\u7f51\u5361\u4f1a\u5c06\u6570\u636e\u5305\u76f4\u63a5\u901a\u8fc7\u8be5\u8282\u70b9\u7f51\u5361\u5bf9\u5916\u53d1\u9001\uff0cL2/L3 \u5c42\u9762\u7684\u8f6c\u53d1\u80fd\u529b\u9700\u8981\u4f9d\u8d56\u5e95\u5c42\u7f51\u7edc\u8bbe\u5907\u3002 \u9700\u8981\u9884\u5148\u5728\u5e95\u5c42\u7f51\u7edc\u8bbe\u5907\u914d\u7f6e\u5bf9\u5e94\u7684\u7f51\u5173\u3001Vlan \u548c\u5b89\u5168\u7b56\u7565\u7b49\u914d\u7f6e\u3002</p> <ol> <li>\u5bf9\u4e8e OpenStack \u7684 VM \u73af\u5883\uff0c\u9700\u8981\u5c06\u5bf9\u5e94\u7f51\u7edc\u7aef\u53e3\u7684 <code>PortSecurity</code> \u5173\u95ed\u3002</li> <li>\u5bf9\u4e8e VMware \u7684 vSwitch \u7f51\u7edc\uff0c\u9700\u8981\u5c06 <code>MAC Address Changes</code>, <code>Forged Transmits</code> \u548c <code>Promiscuous Mode Operation</code> \u8bbe\u7f6e\u4e3a <code>allow</code>\u3002</li> <li>\u5bf9\u4e8e Hyper-V \u865a\u62df\u5316\uff0c\u9700\u8981\u5f00\u542f\u865a\u62df\u673a\u7f51\u5361\u9ad8\u7ea7\u529f\u80fd\u4e2d\u7684 <code>MAC Address Spoofing</code>\u3002</li> <li>\u516c\u6709\u4e91\uff0c\u4f8b\u5982 AWS\u3001GCE\u3001\u963f\u91cc\u4e91\u7b49\u7531\u4e8e\u4e0d\u652f\u6301\u7528\u6237\u81ea\u5b9a\u4e49 Mac \u65e0\u6cd5\u652f\u6301 Macvlan \u6a21\u5f0f\u7f51\u7edc\u3002</li> <li>\u7531\u4e8e Macvlan \u672c\u8eab\u7684\u9650\u5236\uff0cMacvlan \u5b50\u63a5\u53e3\u65e0\u6cd5\u8bbf\u95ee\u7236\u63a5\u53e3\u5730\u5740\u3002</li> <li>\u5982\u679c\u7269\u7406\u7f51\u5361\u5bf9\u5e94\u4ea4\u6362\u673a\u63a5\u53e3\u4e3a Trunk \u6a21\u5f0f\uff0c\u9700\u8981\u5728\u8be5\u7f51\u5361\u4e0a\u521b\u5efa\u5b50\u63a5\u53e3\u518d\u63d0\u4f9b\u7ed9 Macvlan \u4f7f\u7528\u3002</li> </ol>"},{"location":"guide/vpc/#vpc_3","title":"\u5f00\u542f VPC \u7f51\u5173\u529f\u80fd","text":"<p>VPC \u7f51\u5173\u529f\u80fd\u9700\u8981\u901a\u8fc7 <code>kube-system</code> \u4e0b\u7684 <code>ovn-vpc-nat-gw-config</code> \u5f00\u542f\uff1a</p> <pre><code>---\nkind: ConfigMap\napiVersion: v1\nmetadata:\nname: ovn-vpc-nat-config\nnamespace: kube-system\ndata:\nimage: 'docker.io/kubeovn/vpc-nat-gateway:v1.13.0' ---\nkind: ConfigMap\napiVersion: v1\nmetadata:\nname: ovn-vpc-nat-gw-config\nnamespace: kube-system\ndata:\nenable-vpc-nat-gw: 'true'\n</code></pre> <ul> <li><code>image</code>: \u7f51\u5173 Pod \u6240\u4f7f\u7528\u7684\u955c\u50cf\u3002</li> <li><code>enable-vpc-nat-gw</code>\uff1a \u63a7\u5236\u662f\u5426\u542f\u7528 VPC \u7f51\u5173\u529f\u80fd\u3002</li> </ul>"},{"location":"guide/vpc/#vpc_4","title":"\u521b\u5efa VPC \u7f51\u5173\u5e76\u914d\u7f6e\u9ed8\u8ba4\u8def\u7531","text":"<pre><code>kind: VpcNatGateway\napiVersion: kubeovn.io/v1\nmetadata:\nname: gw1\nspec:\nvpc: test-vpc-1\nsubnet: net1\nlanIp: 10.0.1.254\nselector:\n- \"kubernetes.io/hostname: kube-ovn-worker\"\n- \"kubernetes.io/os: linux\"\nexternalSubnets:\n- ovn-vpc-external-network\n</code></pre> <ul> <li><code>vpc</code>\uff1a\u8be5 VpcNatGateway \u6240\u5c5e\u7684 VPC\u3002</li> <li><code>subnet</code>\uff1a\u4e3a VPC \u5185\u67d0\u4e2a Subnet \u540d\uff0cVPC \u7f51\u5173 Pod \u4f1a\u5728\u8be5\u5b50\u7f51\u4e0b\u7528 <code>lanIp</code> \u6765\u8fde\u63a5\u79df\u6237\u7f51\u7edc\u3002</li> <li><code>lanIp</code>\uff1a<code>subnet</code> \u5185\u67d0\u4e2a\u672a\u88ab\u4f7f\u7528\u7684 IP\uff0cVPC \u7f51\u5173 Pod \u6700\u7ec8\u4f1a\u4f7f\u7528\u8be5 Pod\u3002\u5f53 VPC \u914d\u7f6e\u8def\u7531\u9700\u8981\u6307\u5411\u5f53\u524d VpcNatGateway \u65f6 <code>nextHopIP</code> \u9700\u8981\u8bbe\u7f6e\u4e3a\u8fd9\u4e2a <code>lanIp</code>\u3002</li> <li><code>selector</code>\uff1aVpcNatGateway Pod \u7684\u8282\u70b9\u9009\u62e9\u5668\uff0c\u683c\u5f0f\u548c Kubernetes \u4e2d\u7684 NodeSelector \u683c\u5f0f\u76f8\u540c\u3002</li> <li><code>externalSubnets</code>\uff1a VPC \u7f51\u5173\u4f7f\u7528\u7684\u5916\u90e8\u7f51\u7edc\uff0c\u5982\u679c\u4e0d\u914d\u7f6e\u5219\u9ed8\u8ba4\u4f7f\u7528 <code>ovn-vpc-external-network</code>\uff0c\u5f53\u524d\u7248\u672c\u53ea\u652f\u6301\u914d\u7f6e\u4e00\u4e2a\u5916\u90e8\u7f51\u7edc\u3002</li> </ul> <p>\u5176\u4ed6\u53ef\u914d\u53c2\u6570\uff1a</p> <ul> <li><code>tolerations</code> : \u4e3a VPC \u7f51\u5173\u914d\u7f6e\u5bb9\u5fcd\u5ea6\uff0c\u5177\u4f53\u914d\u7f6e\u53c2\u8003 \u6c61\u70b9\u548c\u5bb9\u5fcd\u5ea6\u3002</li> <li><code>affinity</code> :  \u4e3a VPC \u7f51\u5173 Pod \u6216\u8282\u70b9\u914d\u7f6e\u4eb2\u548c\u6027\uff0c\u5177\u4f53\u8bbe\u7f6e\u53c2\u8003 \u4eb2\u548c\u6027\u4e0e\u53cd\u4eb2\u548c\u6027\u3002</li> </ul>"},{"location":"guide/vpc/#eip","title":"\u521b\u5efa EIP","text":"<p>EIP \u4e3a\u5916\u90e8\u7f51\u7edc\u6bb5\u7684\u67d0\u4e2a IP \u5206\u914d\u7ed9 VPC \u7f51\u5173\u540e\u53ef\u8fdb\u884c DNAT\uff0cSNAT \u548c\u6d6e\u52a8 IP \u64cd\u4f5c\u3002</p> <p>\u968f\u673a\u5206\u914d\u4e00\u4e2a\u5730\u5740\u7ed9 EIP\uff1a</p> <pre><code>kind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-random\nspec:\nnatGwDp: gw1\n</code></pre> <p>\u56fa\u5b9a EIP \u5730\u5740\u5206\u914d\uff1a</p> <pre><code>kind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-static\nspec:\nnatGwDp: gw1\nv4ip: 10.0.1.111\n</code></pre> <p>\u6307\u5b9a EIP \u6240\u5728\u7684\u5916\u90e8\u7f51\u7edc\uff1a</p> <pre><code>kind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-random\nspec:\nnatGwDp: gw1\nexternalSubnet: ovn-vpc-external-network\n</code></pre> <ul> <li><code>externalSubnet</code>\uff1a EIP \u6240\u5728\u5916\u90e8\u7f51\u7edc\u540d\u79f0\uff0c\u5982\u679c\u4e0d\u6307\u5b9a\u5219\u9ed8\u8ba4\u4e3a <code>ovn-vpc-external-network</code>\uff0c\u5982\u679c\u6307\u5b9a\u5219\u5fc5\u987b\u4e3a\u6240\u5728 VPC \u7f51\u5173\u7684 <code>externalSubnets</code> \u4e2d\u7684\u4e00\u4e2a\u3002</li> </ul>"},{"location":"guide/vpc/#dnat","title":"\u521b\u5efa DNAT \u89c4\u5219","text":"<p>\u901a\u8fc7 DNAT \u89c4\u5219\uff0c\u5916\u90e8\u53ef\u4ee5\u901a\u8fc7\u4e00\u4e2a EIP \u52a0\u7aef\u53e3\u7684\u65b9\u5f0f\u6765\u8bbf\u95ee VPC \u5185\u7684\u4e00\u4e2a IP \u548c\u7aef\u53e3\u3002</p> <pre><code>kind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eipd01\nspec:\nnatGwDp: gw1\n\n---\nkind: IptablesDnatRule\napiVersion: kubeovn.io/v1\nmetadata:\nname: dnat01\nspec:\neip: eipd01 externalPort: '8888'\ninternalIp: 10.0.1.10\ninternalPort: '80'\nprotocol: tcp\n</code></pre>"},{"location":"guide/vpc/#snat","title":"\u521b\u5efa SNAT \u89c4\u5219","text":"<p>\u901a\u8fc7 SNAT \u89c4\u5219\uff0cVPC \u5185\u7684 Pod \u8bbf\u95ee\u5916\u90e8\u7684\u5730\u5740\u65f6\u5c06\u4f1a\u901a\u8fc7\u5bf9\u5e94 EIP \u8fdb\u884c SNAT\u3002</p> <pre><code>---\nkind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eips01\nspec:\nnatGwDp: gw1\n---\nkind: IptablesSnatRule\napiVersion: kubeovn.io/v1\nmetadata:\nname: snat01\nspec:\neip: eips01\ninternalCIDR: 10.0.1.0/24\n</code></pre>"},{"location":"guide/vpc/#ip","title":"\u521b\u5efa\u6d6e\u52a8 IP","text":"<p>\u901a\u8fc7\u6d6e\u52a8 IP \u89c4\u5219\uff0cVPC \u5185\u7684\u4e00\u4e2a IP \u4f1a\u548c EIP \u8fdb\u884c\u5b8c\u5168\u6620\u5c04\uff0c\u5916\u90e8\u53ef\u4ee5\u901a\u8fc7\u8fd9\u4e2a EIP \u8bbf\u95ee VPC \u5185\u7684 IP\uff0cVPC \u5185\u7684\u8fd9\u4e2a IP \u8bbf\u95ee\u5916\u90e8\u5730\u5740\u65f6\u4e5f\u4f1a SNAT \u6210\u8fd9\u4e2a EIP\u3002</p> <pre><code>---\nkind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eipf01\nspec:\nnatGwDp: gw1\n\n---\nkind: IptablesFIPRule\napiVersion: kubeovn.io/v1\nmetadata:\nname: fip01\nspec:\neip: eipf01\ninternalIp: 10.0.1.5\n</code></pre>"},{"location":"guide/vpc/#_2","title":"\u81ea\u5b9a\u4e49\u8def\u7531","text":"<p>\u5728\u81ea\u5b9a\u4e49 VPC \u5185\uff0c\u7528\u6237\u53ef\u4ee5\u81ea\u5b9a\u4e49\u7f51\u7edc\u5185\u90e8\u7684\u8def\u7531\u89c4\u5219\uff0c\u7ed3\u5408\u7f51\u5173\u5b9e\u73b0\u66f4\u7075\u6d3b\u7684\u8f6c\u53d1\u3002 Kube-OVN \u652f\u6301\u9759\u6001\u8def\u7531\u548c\u66f4\u4e3a\u7075\u6d3b\u7684\u7b56\u7565\u8def\u7531\u3002</p>"},{"location":"guide/vpc/#_3","title":"\u9759\u6001\u8def\u7531","text":"<pre><code>kind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: test-vpc-1\nspec:\nstaticRoutes:\n- cidr: 0.0.0.0/0\nnextHopIP: 10.0.1.254\npolicy: policyDst\n- cidr: 172.31.0.0/24\nnextHopIP: 10.0.1.253\npolicy: policySrc\nrouteTable: \"rtb1\"\n</code></pre> <ul> <li><code>policy</code>: \u652f\u6301\u76ee\u7684\u5730\u5740\u8def\u7531 <code>policyDst</code> \u548c\u6e90\u5730\u5740\u8def\u7531 <code>policySrc</code>\u3002</li> <li>\u5f53\u8def\u7531\u89c4\u5219\u5b58\u5728\u91cd\u53e0\u65f6\uff0cCIDR \u63a9\u7801\u8f83\u957f\u7684\u89c4\u5219\u4f18\u5148\u7ea7\u66f4\u9ad8\uff0c\u82e5\u63a9\u7801\u957f\u5ea6\u76f8\u540c\u5219\u76ee\u7684\u5730\u5740\u8def\u7531\u4f18\u5148\u4e8e\u6e90\u5730\u5740\u8def\u7531\u3002</li> <li><code>routeTable</code>: \u53ef\u6307\u5b9a\u9759\u6001\u8def\u7531\u6240\u5728\u7684\u8def\u7531\u8868\uff0c\u9ed8\u8ba4\u5728\u4e3b\u8def\u7531\u8868\u3002\u5b50\u7f51\u5173\u8054\u8def\u7531\u8868\u8bf7\u53c2\u8003\u521b\u5efa\u5b50\u7f51</li> </ul>"},{"location":"guide/vpc/#_4","title":"\u7b56\u7565\u8def\u7531","text":"<p>\u9488\u5bf9\u9759\u6001\u8def\u7531\u5339\u914d\u7684\u6d41\u91cf\uff0c\u53ef\u901a\u8fc7\u7b56\u7565\u8def\u7531\u8fdb\u884c\u66f4\u7ec6\u7c92\u5ea6\u7684\u63a7\u5236\u3002\u7b56\u7565\u8def\u7531\u63d0\u4f9b\u4e86\u66f4\u7cbe\u786e\u7684\u5339\u914d\u89c4\u5219\uff0c\u4f18\u5148\u7ea7\u63a7\u5236 \u548c\u66f4\u591a\u7684\u8f6c\u53d1\u52a8\u4f5c\u3002\u8be5\u529f\u80fd\u4e3a OVN \u5185\u90e8\u903b\u8f91\u8def\u7531\u5668\u7b56\u7565\u529f\u80fd\u7684\u4e00\u4e2a\u5bf9\u5916\u66b4\u9732\uff0c\u66f4\u591a\u4f7f\u7528\u4fe1\u606f\u8bf7\u53c2\u8003 Logical Router Policy\u3002</p> <p>\u7b80\u5355\u793a\u4f8b\u5982\u4e0b\uff1a</p> <pre><code>kind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: test-vpc-1\nspec:\npolicyRoutes:\n- action: drop\nmatch: ip4.src==10.0.1.0/24 &amp;&amp; ip4.dst==10.0.1.250\npriority: 11\n- action: reroute\nmatch: ip4.src==10.0.1.0/24\nnextHopIP: 10.0.1.252\npriority: 10\n</code></pre>"},{"location":"guide/vpc/#_5","title":"\u81ea\u5b9a\u4e49\u5185\u90e8\u8d1f\u8f7d\u5747\u8861\u89c4\u5219","text":"<p>Kubernetes \u672c\u8eab\u63d0\u4f9b\u7684 Service \u80fd\u529b\u53ef\u4ee5\u5b8c\u6210\u5185\u90e8\u8d1f\u8f7d\u5747\u8861\u7684\u529f\u80fd\uff0c\u4f46\u662f\u53d7\u9650\u4e8e Kubernetes \u5b9e\u73b0\uff0c Service \u7684 IP \u5730\u5740\u662f\u5168\u5c40\u5206\u914d\u4e14\u4e0d\u80fd\u91cd\u590d\u3002\u5bf9\u4e8e VPC \u7684\u4f7f\u7528\u573a\u666f\uff0c\u7528\u6237\u5e0c\u671b\u80fd\u81ea\u5b9a\u4e49\u5185\u90e8\u8d1f\u8f7d\u5747\u8861\u7684\u5730\u5740 \u8303\u56f4\uff0c\u4e0d\u540c VPC \u4e0b\u7684\u8d1f\u8f7d\u5747\u8861\u5730\u5740\u53ef\u80fd\u91cd\u53e0\uff0cKubernetes \u5185\u7f6e\u7684 Service \u529f\u80fd\u65e0\u6cd5\u5b8c\u5168\u6ee1\u8db3\u3002</p> <p>\u9488\u5bf9\u8fd9\u7c7b\u573a\u666f\uff0cKube-OVN \u63d0\u4f9b\u4e86 <code>SwitchLBRule</code> \u8d44\u6e90\uff0c\u7528\u6237\u53ef\u4ee5\u81ea\u5b9a\u4e49\u5185\u90e8\u8d1f\u8f7d\u5747\u8861\u7684\u5730\u5740\u8303\u56f4\u3002</p> <p>\u4e00\u4e2a `SwitchLBRule`` \u4f8b\u5b50\u5982\u4e0b\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: SwitchLBRule\nmetadata:\nname:  cjh-slr-nginx\nspec:\nvip: 1.1.1.1\nsessionAffinity: ClientIP\nnamespace: default\nselector:\n- app:nginx\nports:\n- name: dns\nport: 8888\ntargetPort: 80\nprotocol: TCP\n</code></pre> <ul> <li><code>vip</code>\uff1a\u81ea\u5b9a\u4e49\u5185\u90e8\u8d1f\u8f7d\u5747\u8861\u7684\u5730\u5740\u3002</li> <li><code>namespace</code>\uff1a\u8d1f\u8f7d\u5747\u8861\u5668\u540e\u7aef Pod \u6240\u5728\u7684 Namespace\u3002</li> <li><code>sessionAffinity</code>\uff1a\u548c Service \u7684 <code>sessionAffinity</code> \u529f\u80fd\u76f8\u540c\u3002</li> <li><code>selector</code>\uff1a\u548c Service \u7684 <code>selector</code> \u529f\u80fd\u76f8\u540c\u3002</li> <li><code>ports</code>\uff1a\u548c Service \u7684 <code>port</code> \u529f\u80fd\u76f8\u540c\u3002</li> </ul> <p>\u67e5\u770b\u90e8\u7f72\u7684\u5185\u90e8\u8d1f\u8f7d\u5747\u8861\u5668\u89c4\u5219\uff1a</p> <pre><code># kubectl get slr\nNAME                VIP         PORT(S)                  SERVICE                             AGE\nvpc-dns-test-cjh2   10.96.0.3   53/UDP,53/TCP,9153/TCP   kube-system/slr-vpc-dns-test-cjh2   88m\n</code></pre>"},{"location":"guide/vpc/#vpc-dns","title":"\u81ea\u5b9a\u4e49 vpc-dns","text":"<p>\u7531\u4e8e\u81ea\u5b9a\u4e49 VPC \u548c\u9ed8\u8ba4 VPC \u7f51\u7edc\u76f8\u4e92\u9694\u79bb\uff0cVPC \u5185 Pod \u65e0\u6cd5\u4f7f\u7528\u9ed8\u8ba4\u7684 coredns \u670d\u52a1\u8fdb\u884c\u57df\u540d\u89e3\u6790\u3002 \u5982\u679c\u5e0c\u671b\u5728\u81ea\u5b9a\u4e49 VPC \u5185\u4f7f\u7528 coredns \u89e3\u6790\u96c6\u7fa4\u5185 Service \u57df\u540d\uff0c\u53ef\u4ee5\u901a\u8fc7 Kube-OVN \u63d0\u4f9b\u7684 vpc-dns \u8d44\u6e90\u6765\u5b9e\u73b0\u3002</p>"},{"location":"guide/vpc/#_6","title":"\u521b\u5efa\u9644\u52a0\u7f51\u5361","text":"<pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: ovn-nad\nnamespace: default\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.0\",\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"ovn-nad.default.ovn\"\n}'\n</code></pre>"},{"location":"guide/vpc/#ovn-default-provider","title":"\u4fee\u6539 ovn-default \u903b\u8f91\u4ea4\u6362\u673a\u7684 provider","text":"<p>\u4fee\u6539 ovn-default \u7684 provider\uff0c\u4e3a\u4e0a\u9762 nad \u914d\u7f6e\u7684 provider <code>ovn-nad.default.ovn</code>\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: ovn-default\nspec:\ncidrBlock: 10.16.0.0/16\ndefault: true\ndisableGatewayCheck: false\ndisableInterConnection: false\nenableDHCP: false\nenableIPv6RA: false\nexcludeIps:\n- 10.16.0.1\ngateway: 10.16.0.1\ngatewayType: distributed\nlogicalGateway: false\nnatOutgoing: true\nprivate: false\nprotocol: IPv4\nprovider: ovn-nad.default.ovn\nvpc: ovn-cluster\n</code></pre>"},{"location":"guide/vpc/#vpc-dns-configmap","title":"\u914d\u7f6e vpc-dns \u7684 ConfigMap","text":"<p>\u5728 kube-system \u547d\u540d\u7a7a\u95f4\u4e0b\u521b\u5efa configmap\uff0c\u914d\u7f6e vpc-dns \u4f7f\u7528\u53c2\u6570\uff0c\u7528\u4e8e\u540e\u9762\u542f\u52a8 vpc-dns \u529f\u80fd\uff1a</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: vpc-dns-config\nnamespace: kube-system\ndata:\ncoredns-vip: 10.96.0.3\nenable-vpc-dns: true\nnad-name: ovn-nad\nnad-provider: ovn-nad.default.ovn\n</code></pre> <ul> <li><code>enable-vpc-dns</code>\uff1a\uff08\u53ef\u7f3a\u7701\uff09<code>true</code> \u542f\u7528\u529f\u80fd\uff0c<code>false</code> \u5173\u95ed\u529f\u80fd\u3002\u9ed8\u8ba4 <code>true</code>\u3002</li> <li><code>coredns-image</code>\uff1a\uff08\u53ef\u7701\u7565\uff09\uff1adns \u90e8\u7f72\u955c\u50cf\u3002\u9ed8\u8ba4\u4e3a\u96c6\u7fa4 coredns \u90e8\u7f72\u7248\u672c\u3002</li> <li><code>coredns-template</code>\uff1a\uff08\u53ef\u7701\u7565\uff09\uff1adns \u90e8\u7f72\u6a21\u677f\u6240\u5728\u7684 URL\u3002\u9ed8\u8ba4\uff1a\u5f53\u524d\u7248\u672c\u4ed3\u5e93\u91cc\u7684 <code>yamls/coredns-template.yaml</code>\u3002</li> <li><code>coredns-vip</code>\uff1a\u4e3a coredns \u63d0\u4f9b lb \u670d\u52a1\u7684 vip\u3002</li> <li><code>nad-name</code>\uff1a\u914d\u7f6e\u7684 <code>network-attachment-definitions</code> \u8d44\u6e90\u540d\u79f0\u3002</li> <li><code>nad-provider</code>\uff1a\u4f7f\u7528\u7684 provider \u540d\u79f0\u3002</li> <li><code>k8s-service-host</code>\uff1a\uff08\u53ef\u7f3a\u7701\uff09 \u7528\u4e8e coredns \u8bbf\u95ee k8s apiserver \u670d\u52a1\u7684 ip\u3002</li> <li><code>k8s-service-port</code>\uff1a\uff08\u53ef\u7f3a\u7701\uff09\u7528\u4e8e coredns \u8bbf\u95ee k8s apiserver \u670d\u52a1\u7684 port\u3002</li> </ul>"},{"location":"guide/vpc/#vpc-dns_1","title":"\u90e8\u7f72 vpc-dns \u4f9d\u8d56\u8d44\u6e90","text":"<pre><code>apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\nlabels:\nkubernetes.io/bootstrapping: rbac-defaults\nname: system:vpc-dns\nrules:\n- apiGroups:\n- \"\"\nresources:\n- endpoints\n- services\n- pods\n- namespaces\nverbs:\n- list\n- watch\n- apiGroups:\n- discovery.k8s.io\nresources:\n- endpointslices\nverbs:\n- list\n- watch\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\nannotations:\nrbac.authorization.kubernetes.io/autoupdate: \"true\"\nlabels:\nkubernetes.io/bootstrapping: rbac-defaults\nname: vpc-dns\nroleRef:\napiGroup: rbac.authorization.k8s.io\nkind: ClusterRole\nname: system:vpc-dns\nsubjects:\n- kind: ServiceAccount\nname: vpc-dns\nnamespace: kube-system\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\nname: vpc-dns\nnamespace: kube-system\n---\napiVersion: v1\nkind: ConfigMap\nmetadata:\nname: vpc-dns-corefile\nnamespace: kube-system\ndata:\nCorefile: |\n.:53 {\nerrors\nhealth {\nlameduck 5s\n}\nready\nkubernetes cluster.local in-addr.arpa ip6.arpa {\npods insecure\nfallthrough in-addr.arpa ip6.arpa\n}\nprometheus :9153\nforward . /etc/resolv.conf {\nprefer_udp\n}\ncache 30\nloop\nreload\nloadbalance\n}\n</code></pre>"},{"location":"guide/vpc/#vpc-dns_2","title":"\u90e8\u7f72 vpc-dns","text":"<pre><code>kind: VpcDns\napiVersion: kubeovn.io/v1\nmetadata:\nname: test-cjh1\nspec:\nvpc: cjh-vpc-1\nsubnet: cjh-subnet-1\n</code></pre> <ul> <li><code>vpc</code>\uff1a \u7528\u4e8e\u90e8\u7f72 dns \u7ec4\u4ef6\u7684 vpc \u540d\u79f0\u3002</li> <li><code>subnet</code>\uff1a\u7528\u4e8e\u90e8\u7f72 dns \u7ec4\u4ef6\u7684\u5b50\u540d\u79f0\u3002</li> </ul> <p>\u67e5\u770b\u8d44\u6e90\u4fe1\u606f\uff1a</p> <pre><code>[root@hci-dev-mst-1 kubeovn]# kubectl get vpc-dns\nNAME        ACTIVE   VPC         SUBNET   \ntest-cjh1   false    cjh-vpc-1   cjh-subnet-1   \ntest-cjh2   true     cjh-vpc-1   cjh-subnet-2 </code></pre> <ul> <li><code>ACTIVE</code>: <code>true</code> \u6210\u529f\u90e8\u7f72\u4e86\u81ea\u5b9a\u4e49 dns \u7ec4\u4ef6\uff0c<code>false</code> \u65e0\u90e8\u7f72</li> </ul>"},{"location":"guide/vpc/#_7","title":"\u9650\u5236","text":"<ul> <li>\u4e00\u4e2a vpc \u4e0b\u53ea\u4f1a\u90e8\u7f72\u4e00\u4e2a\u81ea\u5b9a\u4e49 dns \u7ec4\u4ef6;</li> <li>\u5f53\u4e00\u4e2a vpc \u4e0b\u914d\u7f6e\u591a\u4e2a vpc-dns \u8d44\u6e90\uff08\u5373\u540c\u4e00\u4e2a vpc \u4e0d\u540c\u7684 subnet\uff09\uff0c\u53ea\u6709\u4e00\u4e2a vpc-dns \u8d44\u6e90\u72b6\u6001 <code>true</code>\uff0c\u5176\u4ed6\u4e3a <code>fasle</code>;</li> <li>\u5f53 <code>true</code> \u7684 vpc-dns \u88ab\u5220\u9664\u6389\uff0c\u4f1a\u83b7\u53d6\u5176\u4ed6 <code>false</code> \u7684 vpc-dns \u8fdb\u884c\u90e8\u7f72\u3002</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"guide/webhook/","title":"Webhook \u4f7f\u7528","text":"<p>\u4f7f\u7528 Webhook \u53ef\u4ee5\u5bf9 Kube-OVN \u5185\u7684 CRD \u8d44\u6e90\u8fdb\u884c\u6821\u9a8c\uff0c\u76ee\u524d Webhook \u4e3b\u8981\u5b8c\u6210 \u56fa\u5b9a IP \u5730\u5740\u51b2\u7a81\u68c0\u6d4b\u548c Subnet CIDR \u7684\u51b2\u7a81\u68c0\u6d4b\uff0c\u5e76\u5728\u8fd9\u7c7b\u8d44\u6e90\u521b\u5efa\u51b2\u7a81\u65f6\u63d0\u793a\u9519\u8bef\u3002</p> <p>\u7531\u4e8e Webhook \u4f1a\u62e6\u622a\u6240\u6709\u7684 Subnet \u548c Pod \u521b\u5efa\u7684\u8bf7\u6c42\uff0c\u56e0\u6b64\u9700\u8981\u5148\u90e8\u7f72 Kube-OVN \u540e\u90e8\u7f72 Webhook \u907f\u514d\u65e0\u6cd5\u521b\u5efa Pod\u3002</p>"},{"location":"guide/webhook/#cert-manager","title":"Cert-Manager \u5b89\u88c5","text":"<p>Webhook \u90e8\u7f72\u9700\u8981\u76f8\u5173\u8bc1\u4e66\u52a0\u5bc6\uff0c\u6211\u4eec\u4f7f\u7528 cert-manager \u751f\u6210\u76f8\u5173\u8bc1\u4e66\uff0c\u6211\u4eec\u9700\u8981\u5728\u90e8\u7f72 Webhook \u524d\u5148\u90e8\u7f72 cert-manager\u3002</p> <p>\u53ef\u4ee5\u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u90e8\u7f72 cert-manager:</p> <pre><code>kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.8.0/cert-manager.yaml\n</code></pre> <p>\u66f4\u591a cert-manager \u4f7f\u7528\u8bf7\u53c2\u8003 cert-manager \u6587\u6863\u3002</p>"},{"location":"guide/webhook/#webhook_1","title":"\u5b89\u88c5 Webhook","text":"<p>\u4e0b\u8f7d Webhook \u5bf9\u5e94\u7684 yaml \u8fdb\u884c\u5b89\u88c5:</p> <pre><code># kubectl apply -f https://raw.githubusercontent.com/kubeovn/kube-ovn/master/yamls/webhook.yaml\ndeployment.apps/kube-ovn-webhook created\nservice/kube-ovn-webhook created\nvalidatingwebhookconfiguration.admissionregistration.k8s.io/kube-ovn-webhook created\ncertificate.cert-manager.io/kube-ovn-webhook-serving-cert created\nissuer.cert-manager.io/kube-ovn-webhook-selfsigned-issuer created\n</code></pre>"},{"location":"guide/webhook/#webhook_2","title":"\u9a8c\u8bc1 Webhook \u751f\u6548","text":"<p>\u67e5\u770b\u5df2\u8fd0\u884c Pod\uff0c\u5f97\u5230 Pod IP <code>10.16.0.15</code>\uff1a</p> <pre><code># kubectl get pod -o wide\nNAME                      READY   STATUS    RESTARTS   AGE     IP           NODE              NOMINATED NODE   READINESS GATES\nstatic-7584848b74-fw9dm   1/1     Running   0          2d13h   10.16.0.15   kube-ovn-worker   &lt;none&gt; </code></pre> <p>\u7f16\u5199 yaml \u521b\u5efa\u76f8\u540c IP \u7684 Pod\uff1a</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nannotations:\novn.kubernetes.io/ip_address: 10.16.0.15\novn.kubernetes.io/mac_address: 00:00:00:53:6B:B6\nlabels:\napp: static\nmanagedFields:\nname: staticip-pod\nnamespace: default\nspec:\ncontainers:\n- image: docker.io/library/nginx:alpine\nimagePullPolicy: IfNotPresent\nname: qatest\n</code></pre> <p>\u4f7f\u7528\u4ee5\u4e0a yaml \u521b\u5efa\u9759\u6001\u5730\u5740 Pod \u7684\u65f6\u5019\uff0c\u63d0\u793a IP \u5730\u5740\u51b2\u7a81\uff1a</p> <pre><code># kubectl apply -f pod-static.yaml\nError from server (annotation ip address 10.16.0.15 is conflict with ip crd static-7584848b74-fw9dm.default 10.16.0.15): error when creating \"pod-static.yaml\": admission webhook \"pod-ip-validaing.kube-ovn.io\" denied the request: annotation ip address 10.16.0.15 is conflict with ip crd static-7584848b74-fw9dm.default 10.16.0.15\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"ops/change-default-subnet/","title":"\u4fee\u6539\u5b50\u7f51 CIDR","text":"<p>\u5982\u679c\u521b\u5efa\u7684\u5b50\u7f51 CIDR \u51b2\u7a81\u6216\u4e0d\u7b26\u5408\u9884\u671f\uff0c\u53ef\u4ee5\u901a\u8fc7\u672c\u6587\u6863\u7684\u6b65\u9aa4\u8fdb\u884c\u4fee\u6539\u3002</p> <p>\u4fee\u6539\u5b50\u7f51 CIDR \u540e\u4e4b\u524d\u521b\u5efa\u7684 Pod \u5c06\u65e0\u6cd5\u6b63\u5e38\u8bbf\u95ee\u7f51\u7edc\u9700\u8981\u8fdb\u884c\u91cd\u5efa\u3002 \u5efa\u8bae\u64cd\u4f5c\u524d\u614e\u91cd\u8003\u8651\u3002\u672c\u6587\u53ea\u9488\u5bf9\u4e1a\u52a1\u5b50\u7f51 CIDR \u66f4\u6539\u8fdb\u884c\u64cd\u4f5c\uff0c\u5982\u9700 \u66f4\u6539 Join \u5b50\u7f51 CIDR \u8bf7\u53c2\u8003\u66f4\u6539 Join \u5b50\u7f51 CIDR\u3002</p>"},{"location":"ops/change-default-subnet/#_1","title":"\u7f16\u8f91\u5b50\u7f51","text":"<p>\u4f7f\u7528 <code>kubectl edit</code> \u4fee\u6539\u5b50\u7f51 <code>cidrBlock</code>\uff0c<code>gateway</code> \u548c <code>excludeIps</code>\u3002</p> <pre><code>kubectl edit subnet test-subnet\n</code></pre>"},{"location":"ops/change-default-subnet/#namespace-pod","title":"\u91cd\u5efa\u8be5\u5b50\u7f51\u7ed1\u5b9a\u7684 Namespace \u4e0b\u6240\u6709 Pod","text":"<p>\u4ee5\u5b50\u7f51\u7ed1\u5b9a <code>test</code> Namespace \u4e3a\u4f8b\uff1a</p> <pre><code>for pod in $(kubectl get pod --no-headers -n \"$ns\" --field-selector spec.restartPolicy=Always -o custom-columns=NAME:.metadata.name,HOST:spec.hostNetwork | awk '{if ($2!=\"true\") print $1}'); do\nkubectl delete pod \"$pod\" -n test --ignore-not-found\ndone\n</code></pre> <p>\u82e5\u53ea\u4f7f\u7528\u4e86\u9ed8\u8ba4\u5b50\u7f51\uff0c\u53ef\u4ee5\u4f7f\u7528\u4e0b\u5217\u547d\u4ee4\u5220\u9664\u6240\u6709\u975e host \u7f51\u7edc\u6a21\u5f0f\u7684 Pod\uff1a</p> <pre><code>for ns in $(kubectl get ns --no-headers -o custom-columns=NAME:.metadata.name); do\nfor pod in $(kubectl get pod --no-headers -n \"$ns\" --field-selector spec.restartPolicy=Always -o custom-columns=NAME:.metadata.name,HOST:spec.hostNetwork | awk '{if ($2!=\"true\") print $1}'); do\nkubectl delete pod \"$pod\" -n \"$ns\" --ignore-not-found\n  done\ndone\n</code></pre>"},{"location":"ops/change-default-subnet/#_2","title":"\u66f4\u6539\u9ed8\u8ba4\u5b50\u7f51\u914d\u7f6e","text":"<p>\u82e5\u4fee\u6539\u7684\u4e3a\u9ed8\u8ba4\u5b50\u7f51\u7684 CIDR \u8fd8\u9700\u8981\u66f4\u6539 <code>kube-ovn-controller</code> Deployment \u7684\u542f\u52a8\u53c2\u6570\uff1a</p> <pre><code>args:\n- --default-cidr=10.17.0.0/16\n- --default-gateway=10.17.0.1\n- --default-exclude-ips=10.17.0.1\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"ops/change-join-subnet/","title":"\u4fee\u6539 Join \u5b50\u7f51 CIDR","text":"<p>\u82e5\u53d1\u73b0\u521b\u5efa\u7684 Join \u5b50\u7f51 CIDR \u51b2\u7a81\u6216\u4e0d\u7b26\u5408\u9884\u671f\uff0c\u53ef\u4ee5\u901a\u8fc7\u672c\u6587\u6863\u8fdb\u884c\u4fee\u6539\u3002</p> <p>\u4fee\u6539 Join \u5b50\u7f51 CIDR \u540e\u4e4b\u524d\u521b\u5efa\u7684 Pod \u5c06\u65e0\u6cd5\u6b63\u5e38\u8bbf\u95ee\u5916\u90e8\u7f51\u7edc\uff0c\u9700\u8981\u7b49\u91cd\u5efa\u5b8c\u6210, \u5efa\u8bae\u524d\u64cd\u4f5c\u65f6\u614e\u91cd\u8003\u8651\u3002</p>"},{"location":"ops/change-join-subnet/#join","title":"\u5220\u9664 Join \u5b50\u7f51","text":"<pre><code>kubectl patch subnet join --type='json' -p '[{\"op\": \"replace\", \"path\": \"/metadata/finalizers\", \"value\": []}]'\nkubectl delete subnet join\n</code></pre>"},{"location":"ops/change-join-subnet/#_1","title":"\u6e05\u7406\u76f8\u5173\u5206\u914d\u4fe1\u606f","text":"<pre><code>kubectl annotate node ovn.kubernetes.io/allocated=false --all --overwrite\n</code></pre>"},{"location":"ops/change-join-subnet/#join_1","title":"\u4fee\u6539 Join \u5b50\u7f51\u76f8\u5173\u4fe1\u606f","text":"<p>\u4fee\u6539 <code>kube-ovn-controller</code> \u5185 Join \u5b50\u7f51\u76f8\u5173\u4fe1\u606f\uff1a</p> <pre><code>kubectl edit deployment -n kube-system kube-ovn-controller\n</code></pre> <p>\u4fee\u6539\u4e0b\u5217\u53c2\u6570\uff1a</p> <pre><code>args:\n- --node-switch-cidr=100.51.0.0/16\n</code></pre> <p>\u91cd\u542f <code>kube-ovn-controller</code> \u91cd\u5efa <code>join</code> \u5b50\u7f51\uff1a</p> <pre><code>kubectl delete pod -n kube-system -lapp=kube-ovn-controller\n</code></pre> <p>\u67e5\u770b\u65b0\u7684 Join \u5b50\u7f51\u4fe1\u606f\uff1a</p> <pre><code># kubectl get subnet\nNAME          PROVIDER   VPC           PROTOCOL   CIDR            PRIVATE   NAT     DEFAULT   GATEWAYTYPE   V4USED   V4AVAILABLE   V6USED   V6AVAILABLE   EXCLUDEIPS\njoin          ovn        ovn-cluster   IPv4       100.51.0.0/16   false     false   false     distributed   2        65531         0        0             [\"100.51.0.1\"]\novn-default   ovn        ovn-cluster   IPv4       10.17.0.0/16    false     true    true      distributed   5        65528         0        0             [\"10.17.0.1\"]\n</code></pre>"},{"location":"ops/change-join-subnet/#ovn0","title":"\u91cd\u65b0\u914d\u7f6e ovn0 \u7f51\u5361\u5730\u5740","text":"<p>\u6bcf\u4e2a\u8282\u70b9\u7684 <code>ovn0</code> \u7f51\u5361\u4fe1\u606f\u9700\u8981\u91cd\u65b0\u66f4\u65b0\uff0c\u53ef\u901a\u8fc7\u91cd\u542f <code>kube-ovn-cni</code> \u6765\u5b8c\u6210\uff1a</p> <pre><code>kubectl delete pod -n kube-system -l app=kube-ovn-cni\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"ops/change-log-level/","title":"\u8c03\u6574\u65e5\u5fd7\u7b49\u7ea7","text":"<p>\u6253\u5f00 <code>kube-ovn.yaml</code>\uff0c\u5728\u670d\u52a1\u542f\u52a8\u811a\u672c\u7684\u53c2\u6570\u5217\u8868\u4e2d\u8bbe\u7f6e log \u7b49\u7ea7\uff0c\u6bd4\u5982\uff1a</p> <pre><code>vi kube-ovn.yaml\n# ...\n- name: kube-ovn-controller\n          image: \"docker.io/kubeovn/kube-ovn:v1.13.0\"\nimagePullPolicy: IfNotPresent\n          args:\n          - /kube-ovn/start-controller.sh\n          - --v=3\n# ...\n# log \u7b49\u7ea7\u8d8a\u9ad8\uff0clog \u5c31\u8d8a\u8be6\u7ec6\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"ops/change-ovn-central-node/","title":"\u66f4\u6362 ovn-central \u8282\u70b9","text":"<p>\u7531\u4e8e <code>ovn-central</code> \u5185\u7684 <code>ovn-nb</code> \u548c <code>ovn-sb</code> \u5206\u522b\u5efa\u7acb\u4e86\u7c7b\u4f3c etcd \u7684 raft \u96c6\u7fa4\uff0c\u56e0\u6b64\u66f4\u6362 <code>ovn-central</code> \u8282\u70b9\u9700\u8981\u989d\u5916\u7684\u64cd\u4f5c\uff0c\u4fdd\u8bc1\u96c6\u7fa4\u72b6\u6001\u7684\u6b63\u786e\u548c\u6570\u636e\u7684\u4e00\u81f4\u3002\u5efa\u8bae\u6bcf\u6b21\u53ea\u5bf9\u4e00\u4e2a\u8282\u70b9\u8fdb\u884c\u4e0a\u4e0b\u7ebf\u5904\u7406\uff0c\u4ee5\u907f\u514d\u96c6\u7fa4\u8fdb\u5165\u4e0d\u53ef\u7528 \u72b6\u6001\uff0c\u5f71\u54cd\u96c6\u7fa4\u6574\u4f53\u7f51\u7edc\u3002</p>"},{"location":"ops/change-ovn-central-node/#ovn-central_1","title":"ovn-central \u8282\u70b9\u4e0b\u7ebf","text":"<p>\u672c\u6587\u6863\u9488\u5bf9\u5982\u4e0b\u7684\u96c6\u7fa4\u60c5\u51b5\uff0c\u4ee5\u4e0b\u7ebf <code>kube-ovn-control-plane2</code> \u8282\u70b9\u4e3a\u4f8b\uff0c\u4ecb\u7ecd\u5982\u4f55\u5c06\u5176\u4ece <code>ovn-central</code> \u96c6\u7fa4\u4e2d\u79fb\u9664\u3002</p> <pre><code># kubectl -n kube-system get pod -o wide | grep central\novn-central-6bf58cbc97-2cdhg                      1/1     Running   0             21m   172.18.0.3   kube-ovn-control-plane    &lt;none&gt;           &lt;none&gt;\novn-central-6bf58cbc97-crmfp                      1/1     Running   0             21m   172.18.0.5   kube-ovn-control-plane2   &lt;none&gt;           &lt;none&gt;\novn-central-6bf58cbc97-lxmpl                      1/1     Running   0             21m   172.18.0.4   kube-ovn-control-plane3   &lt;none&gt;           &lt;none&gt;\n</code></pre>"},{"location":"ops/change-ovn-central-node/#ovn-nb","title":"\u4e0b\u7ebf ovn-nb \u96c6\u7fa4\u5185\u5bf9\u5e94\u8282\u70b9","text":"<p>\u9996\u5148\u67e5\u770b\u8282\u70b9\u5728\u96c6\u7fa4\u5185\u7684 ID\uff0c\u4ee5\u4fbf\u540e\u7eed\u64cd\u4f5c\u3002</p> <pre><code># kubectl ko nb status\n1b9a\nName: OVN_Northbound\nCluster ID: 32ca (32ca07fb-739b-4257-b510-12fa18e7cce8)\nServer ID: 1b9a (1b9a5d76-e69b-410c-8085-39943d0cd38c)\nAddress: tcp:[172.18.0.3]:6643\nStatus: cluster member\nRole: leader\nTerm: 1\nLeader: self\nVote: self\n\nLast Election started 2135194 ms ago, reason: timeout\nLast Election won: 2135188 ms ago\nElection timer: 5000\nLog: [135, 135]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: &lt;-d64b -&gt;d64b &lt;-4984 -&gt;4984\nDisconnections: 0\nServers:\n    4984 (4984 at tcp:[172.18.0.4]:6643) next_index=135 match_index=134 last msg 1084 ms ago\n    1b9a (1b9a at tcp:[172.18.0.3]:6643) (self) next_index=2 match_index=134\nd64b (d64b at tcp:[172.18.0.5]:6643) next_index=135 match_index=134 last msg 1084 ms ago\nstatus: ok\n</code></pre> <p><code>kube-ovn-control-plane2</code> \u5bf9\u5e94\u8282\u70b9 IP \u4e3a <code>172.18.0.5</code>\uff0c\u96c6\u7fa4\u5185\u5bf9\u5e94\u7684 ID \u4e3a <code>d64b</code>\u3002\u63a5\u4e0b\u6765\u4ece ovn-nb \u96c6\u7fa4\u4e2d\u8e22\u51fa\u8be5\u8282\u70b9\uff1a</p> <pre><code># kubectl ko nb kick d64b\nstarted removal\n</code></pre> <p>\u786e\u8ba4\u8282\u70b9\u8e22\u51fa\u6210\u529f\uff1a</p> <pre><code># kubectl ko nb status\n1b9a\nName: OVN_Northbound\nCluster ID: 32ca (32ca07fb-739b-4257-b510-12fa18e7cce8)\nServer ID: 1b9a (1b9a5d76-e69b-410c-8085-39943d0cd38c)\nAddress: tcp:[172.18.0.3]:6643\nStatus: cluster member\nRole: leader\nTerm: 1\nLeader: self\nVote: self\n\nLast Election started 2297649 ms ago, reason: timeout\nLast Election won: 2297643 ms ago\nElection timer: 5000\nLog: [136, 136]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: &lt;-4984 -&gt;4984\nDisconnections: 2\nServers:\n    4984 (4984 at tcp:[172.18.0.4]:6643) next_index=136 match_index=135 last msg 1270 ms ago\n    1b9a (1b9a at tcp:[172.18.0.3]:6643) (self) next_index=2 match_index=135\nstatus: ok\n</code></pre>"},{"location":"ops/change-ovn-central-node/#ovn-sb","title":"\u4e0b\u7ebf ovn-sb \u96c6\u7fa4\u5185\u5bf9\u5e94\u8282\u70b9","text":"<p>\u63a5\u4e0b\u6765\u9700\u8981\u64cd\u4f5c ovn-sb \u96c6\u7fa4\uff0c\u9996\u5148\u67e5\u770b\u8282\u70b9\u5728\u96c6\u7fa4\u5185\u7684 ID\uff0c\u4ee5\u4fbf\u540e\u7eed\u64cd\u4f5c\uff1a</p> <pre><code>kubectl ko sb status\n3722\nName: OVN_Southbound\nCluster ID: d4bd (d4bd37a4-0400-499f-b4df-b4fd389780f0)\nServer ID: 3722 (3722d5ae-2ced-4820-a6b2-8b744d11fb3e)\nAddress: tcp:[172.18.0.3]:6644\nStatus: cluster member\nRole: leader\nTerm: 1\nLeader: self\nVote: self\n\nLast Election started 2395317 ms ago, reason: timeout\nLast Election won: 2395316 ms ago\nElection timer: 5000\nLog: [130, 130]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: &lt;-e9f7 -&gt;e9f7 &lt;-6e84 -&gt;6e84\nDisconnections: 0\nServers:\n    e9f7 (e9f7 at tcp:[172.18.0.5]:6644) next_index=130 match_index=129 last msg 1006 ms ago\n    6e84 (6e84 at tcp:[172.18.0.4]:6644) next_index=130 match_index=129 last msg 1004 ms ago\n    3722 (3722 at tcp:[172.18.0.3]:6644) (self) next_index=2 match_index=129\nstatus: ok\n</code></pre> <p><code>kube-ovn-control-plane2</code> \u5bf9\u5e94\u8282\u70b9 IP \u4e3a <code>172.18.0.5</code>\uff0c\u96c6\u7fa4\u5185\u5bf9\u5e94\u7684 ID \u4e3a <code>e9f7</code>\u3002\u63a5\u4e0b\u6765\u4ece ovn-sb \u96c6\u7fa4\u4e2d\u8e22\u51fa\u8be5\u8282\u70b9\uff1a</p> <pre><code># kubectl ko sb kick e9f7\nstarted removal\n</code></pre> <p>\u786e\u8ba4\u8282\u70b9\u8e22\u51fa\u6210\u529f\uff1a</p> <pre><code># kubectl ko sb status\n3722\nName: OVN_Southbound\nCluster ID: d4bd (d4bd37a4-0400-499f-b4df-b4fd389780f0)\nServer ID: 3722 (3722d5ae-2ced-4820-a6b2-8b744d11fb3e)\nAddress: tcp:[172.18.0.3]:6644\nStatus: cluster member\nRole: leader\nTerm: 1\nLeader: self\nVote: self\n\nLast Election started 2481636 ms ago, reason: timeout\nLast Election won: 2481635 ms ago\nElection timer: 5000\nLog: [131, 131]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: &lt;-6e84 -&gt;6e84\nDisconnections: 2\nServers:\n    6e84 (6e84 at tcp:[172.18.0.4]:6644) next_index=131 match_index=130 last msg 642 ms ago\n    3722 (3722 at tcp:[172.18.0.3]:6644) (self) next_index=2 match_index=130\nstatus: ok\n</code></pre>"},{"location":"ops/change-ovn-central-node/#ovn-central_2","title":"\u5220\u9664\u8282\u70b9\u6807\u7b7e\uff0c\u5e76\u7f29\u5bb9 ovn-central","text":"<p>\u6ce8\u610f\u9700\u5728 ovn-central \u73af\u5883\u53d8\u91cf <code>NODE_IPS</code> \u7684\u8282\u70b9\u5730\u5740\u4e2d\u5220\u9664\u4e0b\u7ebf\u8282\u70b9\u3002</p> <pre><code>kubectl label node kube-ovn-control-plane2 kube-ovn/role-\nkubectl scale deployment -n kube-system ovn-central --replicas=2\nkubectl set env deployment/ovn-central -n kube-system NODE_IPS=\"172.18.0.3,172.18.0.4\"\nkubectl rollout status deployment/ovn-central -n kube-system </code></pre>"},{"location":"ops/change-ovn-central-node/#ovn-central_3","title":"\u4fee\u6539\u5176\u4ed6\u7ec4\u4ef6\u8fde\u63a5 ovn-central \u5730\u5740","text":"<p>\u4fee\u6539 <code>ovs-ovn</code> \u5185\u8fde\u63a5\u4fe1\u606f\uff0c\u5220\u9664\u4e0b\u7ebf\u8282\u70b9\u5730\u5740\u3002</p> <pre><code># kubectl set env daemonset/ovs-ovn -n kube-system OVN_DB_IPS=\"172.18.0.3,172.18.0.4\"\ndaemonset.apps/ovs-ovn env updated\n# kubectl delete pod -n kube-system -lapp=ovs\npod \"ovs-ovn-4f6jc\" deleted\npod \"ovs-ovn-csn2w\" deleted\npod \"ovs-ovn-mpbmb\" deleted\n</code></pre> <p>\u4fee\u6539 <code>kube-ovn-controller</code> \u5185\u8fde\u63a5\u4fe1\u606f\uff0c\u5220\u9664\u4e0b\u7ebf\u8282\u70b9\u5730\u5740\u3002</p> <pre><code># kubectl set env deployment/kube-ovn-controller -n kube-system OVN_DB_IPS=\"172.18.0.3,172.18.0.4\"\ndeployment.apps/kube-ovn-controller env updated\n\n# kubectl rollout status deployment/kube-ovn-controller -n kube-system\nWaiting for deployment \"kube-ovn-controller\" rollout to finish: 1 of 3 updated replicas are available...\nWaiting for deployment \"kube-ovn-controller\" rollout to finish: 2 of 3 updated replicas are available...\ndeployment \"kube-ovn-controller\" successfully rolled out\n</code></pre>"},{"location":"ops/change-ovn-central-node/#_1","title":"\u6e05\u7406\u8282\u70b9","text":"<p>\u5220\u9664 <code>kube-ovn-control-plane2</code> \u8282\u70b9\u5185\u7684\u6570\u636e\u5e93\u6587\u4ef6\uff0c\u907f\u514d\u91cd\u590d\u6dfb\u52a0\u8282\u70b9\u65f6\u53d1\u751f\u5f02\u5e38\uff1a</p> <pre><code>rm -rf /etc/origin/ovn\n</code></pre> <p>\u5982\u9700\u5c06\u8282\u70b9\u4ece\u6574\u4e2a Kubernetes \u96c6\u7fa4\u4e0b\u7ebf\uff0c\u8fd8\u9700\u7ee7\u7eed\u53c2\u8003\u5220\u9664\u5de5\u4f5c\u8282\u70b9\u8fdb\u884c\u64cd\u4f5c\u3002</p>"},{"location":"ops/change-ovn-central-node/#ovn-central_4","title":"ovn-central \u8282\u70b9\u4e0a\u7ebf","text":"<p>\u4e0b\u5217\u6b65\u9aa4\u4f1a\u5c06\u4e00\u4e2a\u65b0\u7684 Kubernetes \u8282\u70b9\u52a0\u5165 <code>ovn-central</code> \u96c6\u7fa4\u3002</p>"},{"location":"ops/change-ovn-central-node/#_2","title":"\u76ee\u5f55\u68c0\u67e5","text":"<p>\u68c0\u67e5\u65b0\u589e\u8282\u70b9\u7684 <code>/etc/origin/ovn</code> \u76ee\u5f55\u4e2d\u662f\u5426\u5b58\u5728 <code>ovnnb_db.db</code> \u6216 <code>ovnsb_db.db</code> \u6587\u4ef6\uff0c\u82e5\u5b58\u5728\u9700\u63d0\u524d\u5220\u9664\uff1a</p> <pre><code>rm -rf /etc/origin/ovn\n</code></pre>"},{"location":"ops/change-ovn-central-node/#ovn-central_5","title":"\u786e\u8ba4\u5f53\u524d ovn-central \u96c6\u7fa4\u72b6\u6001\u6b63\u5e38","text":"<p>\u82e5\u5f53\u524d <code>ovn-central</code> \u96c6\u7fa4\u72b6\u6001\u5df2\u7ecf\u5f02\u5e38\uff0c\u65b0\u589e\u8282\u70b9\u53ef\u80fd\u5bfc\u81f4\u6295\u7968\u9009\u4e3e\u65e0\u6cd5\u8fc7\u534a\u6570\uff0c\u5f71\u54cd\u540e\u7eed\u64cd\u4f5c\u3002</p> <pre><code># kubectl ko nb status\n1b9a\nName: OVN_Northbound\nCluster ID: 32ca (32ca07fb-739b-4257-b510-12fa18e7cce8)\nServer ID: 1b9a (1b9a5d76-e69b-410c-8085-39943d0cd38c)\nAddress: tcp:[172.18.0.3]:6643\nStatus: cluster member\nRole: leader\nTerm: 44\nLeader: self\nVote: self\n\nLast Election started 1855739 ms ago, reason: timeout\nLast Election won: 1855729 ms ago\nElection timer: 5000\nLog: [147, 147]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: -&gt;4984 &lt;-4984\nDisconnections: 0\nServers:\n    4984 (4984 at tcp:[172.18.0.4]:6643) next_index=147 match_index=146 last msg 367 ms ago\n    1b9a (1b9a at tcp:[172.18.0.3]:6643) (self) next_index=140 match_index=146\nstatus: ok\n\n# kubectl ko sb status\n3722\nName: OVN_Southbound\nCluster ID: d4bd (d4bd37a4-0400-499f-b4df-b4fd389780f0)\nServer ID: 3722 (3722d5ae-2ced-4820-a6b2-8b744d11fb3e)\nAddress: tcp:[172.18.0.3]:6644\nStatus: cluster member\nRole: leader\nTerm: 33\nLeader: self\nVote: self\n\nLast Election started 1868589 ms ago, reason: timeout\nLast Election won: 1868579 ms ago\nElection timer: 5000\nLog: [142, 142]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: -&gt;6e84 &lt;-6e84\nDisconnections: 0\nServers:\n    6e84 (6e84 at tcp:[172.18.0.4]:6644) next_index=142 match_index=141 last msg 728 ms ago\n    3722 (3722 at tcp:[172.18.0.3]:6644) (self) next_index=134 match_index=141\nstatus: ok\n</code></pre>"},{"location":"ops/change-ovn-central-node/#_3","title":"\u7ed9\u8282\u70b9\u589e\u52a0\u6807\u7b7e\u5e76\u6269\u5bb9","text":"<p>\u6ce8\u610f\u9700\u5728 ovn-central \u73af\u5883\u53d8\u91cf <code>NODE_IPS</code> \u7684\u8282\u70b9\u5730\u5740\u4e2d\u589e\u52a0\u4e0a\u7ebf\u8282\u70b9\u5730\u5740\u3002</p> <pre><code>kubectl label node kube-ovn-control-plane2 kube-ovn/role=master\nkubectl scale deployment -n kube-system ovn-central --replicas=3\nkubectl set env deployment/ovn-central -n kube-system NODE_IPS=\"172.18.0.3,172.18.0.4,172.18.0.5\"\nkubectl rollout status deployment/ovn-central -n kube-system\n</code></pre>"},{"location":"ops/change-ovn-central-node/#ovn-central_6","title":"\u4fee\u6539\u5176\u4ed6\u7ec4\u4ef6\u8fde\u63a5 ovn-central \u5730\u5740","text":"<p>\u4fee\u6539 <code>ovs-ovn</code> \u5185\u8fde\u63a5\u4fe1\u606f\uff0c\u589e\u52a0\u4e0a\u7ebf\u8282\u70b9\u5730\u5740\uff1a</p> <pre><code># kubectl set env daemonset/ovs-ovn -n kube-system OVN_DB_IPS=\"172.18.0.3,172.18.0.4,172.18.0.5\"\ndaemonset.apps/ovs-ovn env updated\n# kubectl delete pod -n kube-system -lapp=ovs\npod \"ovs-ovn-4f6jc\" deleted\npod \"ovs-ovn-csn2w\" deleted\npod \"ovs-ovn-mpbmb\" deleted\n</code></pre> <p>\u4fee\u6539 <code>kube-ovn-controller</code> \u5185\u8fde\u63a5\u4fe1\u606f\uff0c\u589e\u52a0\u4e0a\u7ebf\u8282\u70b9\u5730\u5740\uff1a</p> <pre><code># kubectl set env deployment/kube-ovn-controller -n kube-system OVN_DB_IPS=\"172.18.0.3,172.18.0.4,172.18.0.5\"\ndeployment.apps/kube-ovn-controller env updated\n\n# kubectl rollout status deployment/kube-ovn-controller -n kube-system\nWaiting for deployment \"kube-ovn-controller\" rollout to finish: 1 of 3 updated replicas are available...\nWaiting for deployment \"kube-ovn-controller\" rollout to finish: 2 of 3 updated replicas are available...\ndeployment \"kube-ovn-controller\" successfully rolled out\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"ops/delete-worker-node/","title":"\u5220\u9664\u5de5\u4f5c\u8282\u70b9","text":"<p>\u5982\u679c\u53ea\u662f\u7b80\u5355\u4ece Kubernetes \u4e2d\u5220\u9664\u8282\u70b9\uff0c\u7531\u4e8e\u8282\u70b9\u4e0a <code>ovs-ovn</code> \u4e2d\u8fd0\u884c\u7684 <code>ovn-controller</code> \u8fdb\u7a0b\u4ecd\u5728\u8fd0\u884c\u4f1a\u5b9a\u671f\u8fde\u63a5 <code>ovn-central</code> \u6ce8\u518c\u76f8\u5173\u7f51\u7edc\u4fe1\u606f\uff0c \u4f1a\u5bfc\u81f4\u989d\u5916\u8d44\u6e90\u6d6a\u8d39\u5e76\u6709\u6f5c\u5728\u7684\u89c4\u5219\u51b2\u7a81\u98ce\u9669\u3002 \u56e0\u6b64\u5728\u4ece Kubernetes \u5185\u5220\u9664\u8282\u70b9\u65f6\uff0c\u8bf7\u6309\u7167\u4e0b\u9762\u7684\u6b65\u9aa4\u6765\u4fdd\u8bc1\u7f51\u7edc\u4fe1\u606f\u53ef\u4ee5\u6b63\u5e38\u88ab\u6e05\u7406\u3002</p> <p>\u8be5\u6587\u6863\u4ecb\u7ecd\u5220\u9664\u5de5\u4f5c\u8282\u70b9\u7684\u6b65\u9aa4\uff0c\u5982\u9700\u66f4\u6362 <code>ovn-central</code> \u6240\u5728\u8282\u70b9\uff0c\u8bf7\u53c2\u8003\u66f4\u6362 ovn-central \u8282\u70b9\u3002</p>"},{"location":"ops/delete-worker-node/#_2","title":"\u9a71\u9010\u8282\u70b9\u4e0a\u6240\u6709\u5bb9\u5668","text":"<pre><code> # kubectl drain kube-ovn-worker --ignore-daemonsets --force\nnode/kube-ovn-worker cordoned\n WARNING: ignoring DaemonSet-managed Pods: kube-system/kube-ovn-cni-zt74b, kube-system/kube-ovn-pinger-5rxfs, kube-system/kube-proxy-jpmnm, kube-system/ovs-ovn-v2kll\n evicting pod kube-system/coredns-64897985d-qsgpt\n evicting pod local-path-storage/local-path-provisioner-5ddd94ff66-llss6\n evicting pod kube-system/kube-ovn-controller-8459db5ff4-94lxb\n pod/kube-ovn-controller-8459db5ff4-94lxb evicted\n pod/coredns-64897985d-qsgpt evicted\n pod/local-path-provisioner-5ddd94ff66-llss6 evicted\n node/kube-ovn-worker drained\n</code></pre>"},{"location":"ops/delete-worker-node/#kubelet-docker","title":"\u505c\u6b62 kubelet \u548c docker","text":"<p>\u8be5\u6b65\u9aa4\u4f1a\u505c\u6b62 <code>ovs-ovn</code> \u5bb9\u5668\uff0c\u4ee5\u907f\u514d\u5411 <code>ovn-central</code> \u8fdb\u884c\u4fe1\u606f\u6ce8\u518c\uff0c\u767b\u5f55\u5230\u5bf9\u5e94\u8282\u70b9\u6267\u884c\u4e0b\u5217\u547d\u4ee4\uff1a</p> <pre><code>systemctl stop kubelet\nsystemctl stop docker\n</code></pre> <p>\u5982\u679c\u4f7f\u7528\u7684 CRI \u4e3a containerd\uff0c\u9700\u8981\u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\u6765\u505c\u6b62 <code>ovs-ovn</code> \u5bb9\u5668\uff1a</p> <pre><code>crictl rm -f $(crictl ps | grep openvswitch | awk '{print $1}')\n</code></pre>"},{"location":"ops/delete-worker-node/#node","title":"\u6e05\u7406 Node \u4e0a\u7684\u6b8b\u7559\u6570\u636e","text":"<pre><code>rm -rf /var/run/openvswitch\nrm -rf /var/run/ovn\nrm -rf /etc/origin/openvswitch/\nrm -rf /etc/origin/ovn/\nrm -rf /etc/cni/net.d/00-kube-ovn.conflist\nrm -rf /etc/cni/net.d/01-kube-ovn.conflist\nrm -rf /var/log/openvswitch\nrm -rf /var/log/ovn\n</code></pre>"},{"location":"ops/delete-worker-node/#kubectl","title":"\u4f7f\u7528 kubectl \u5220\u9664\u8282\u70b9","text":"<pre><code>kubectl delete no kube-ovn-01\n</code></pre>"},{"location":"ops/delete-worker-node/#ovn-sb","title":"\u68c0\u67e5\u5bf9\u5e94\u8282\u70b9\u662f\u5426\u4ece ovn-sb \u4e2d\u5220\u9664","text":"<p>\u4e0b\u9762\u7684\u793a\u4f8b\u4e3a <code>kube-ovn-worker</code> \u4f9d\u7136\u672a\u88ab\u5220\u9664\uff1a</p> <pre><code># kubectl ko sbctl show\nChassis \"b0564934-5a0d-4804-a4c0-476c93596a17\"\nhostname: kube-ovn-worker\n  Encap geneve\n      ip: \"172.18.0.2\"\noptions: {csum=\"true\"}\nPort_Binding kube-ovn-pinger-5rxfs.kube-system\nChassis \"6a29de7e-d731-4eaf-bacd-2f239ee52b28\"\nhostname: kube-ovn-control-plane\n  Encap geneve\n      ip: \"172.18.0.3\"\noptions: {csum=\"true\"}\nPort_Binding coredns-64897985d-nbfln.kube-system\n  Port_Binding node-kube-ovn-control-plane\n  Port_Binding local-path-provisioner-5ddd94ff66-h4tn9.local-path-storage\n  Port_Binding kube-ovn-pinger-hf2p6.kube-system\n  Port_Binding coredns-64897985d-fhwlw.kube-system\n</code></pre>"},{"location":"ops/delete-worker-node/#chassis","title":"\u82e5\u8282\u70b9\u5bf9\u5e94\u7684 chassis \u4f9d\u7136\u5b58\u5728\uff0c\u624b\u52a8\u8fdb\u884c\u5220\u9664","text":"<p>uuid \u4e3a\u4e4b\u524d\u547d\u4ee4\u6240\u67e5\u51fa\u7684 Chassis \u5bf9\u5e94 id\uff1a</p> <pre><code># kubectl ko sbctl chassis-del b0564934-5a0d-4804-a4c0-476c93596a17\n# kubectl ko sbctl show\nChassis \"6a29de7e-d731-4eaf-bacd-2f239ee52b28\"\nhostname: kube-ovn-control-plane\n  Encap geneve\n      ip: \"172.18.0.3\"\noptions: {csum=\"true\"}\nPort_Binding coredns-64897985d-nbfln.kube-system\n  Port_Binding node-kube-ovn-control-plane\n  Port_Binding local-path-provisioner-5ddd94ff66-h4tn9.local-path-storage\n  Port_Binding kube-ovn-pinger-hf2p6.kube-system\n  Port_Binding coredns-64897985d-fhwlw.kube-system\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"ops/faq/","title":"\u5176\u4ed6\u5e38\u89c1\u95ee\u9898","text":""},{"location":"ops/faq/#arm","title":"\u9e92\u9e9f ARM \u7cfb\u7edf\u8de8\u4e3b\u673a\u5bb9\u5668\u8bbf\u95ee\u95f4\u6b47\u5931\u8d25","text":""},{"location":"ops/faq/#_2","title":"\u73b0\u8c61","text":"<p>\u9e92\u9e9f ARM \u7cfb\u7edf\u548c\u90e8\u5206\u56fd\u4ea7\u5316\u7f51\u5361 offload \u914d\u5408\u5b58\u5728\u95ee\u9898\uff0c\u4f1a\u5bfc\u81f4\u5bb9\u5668\u7f51\u7edc\u95f4\u6b47\u6545\u969c\u3002</p> <p>\u4f7f\u7528 <code>netstat</code> \u786e\u8ba4\u95ee\u9898\uff1a</p> <pre><code># netstat -us\nIcmpMsg:\n    InType0: 22\nInType3: 24\nInType8: 117852\nOutType0: 117852\nOutType3: 29\nOutType8: 22\nUdp:\n    3040636 packets received\n    0 packets to unknown port received.\n    4 packet receive errors\n    602 packets sent\n    0 receive buffer errors\n    0 send buffer errors\n    InCsumErrors: 4\nUdpLite:\nIpExt:\n    InBcastPkts: 10244\nInOctets: 4446320361\nOutOctets: 1496815600\nInBcastOctets: 3095950\nInNoECTPkts: 7683903\n</code></pre> <p>\u82e5\u5b58\u5728 <code>InCsumErrors</code>\uff0c\u4e14\u968f\u7740\u8bbf\u95ee\u5931\u8d25\u589e\u52a0\uff0c\u53ef\u786e\u8ba4\u662f\u8be5\u95ee\u9898\u3002</p>"},{"location":"ops/faq/#_3","title":"\u89e3\u51b3\u65b9\u6cd5","text":"<p>\u6839\u672c\u89e3\u51b3\u9700\u8981\u548c\u9e92\u9e9f\u4ee5\u53ca\u5bf9\u5e94\u7f51\u5361\u5382\u5546\u6c9f\u901a\uff0c\u66f4\u65b0\u7cfb\u7edf\u548c\u9a71\u52a8\u3002\u4e34\u65f6\u89e3\u51b3\u53ef\u5148\u5173\u95ed\u7269\u7406 \u7f51\u5361\u7684 <code>tx offload</code> \u4f46\u662f\u4f1a\u5bfc\u81f4 tcp \u6027\u80fd\u6709\u8f83\u660e\u663e\u4e0b\u964d\u3002</p> <pre><code>ethtool -K eth0 tx off\n</code></pre> <p>\u7ecf\u793e\u533a\u53cd\u9988\u4f7f\u7528 <code>4.19.90-25.16.v2101</code> \u5185\u6838\u540e\u53ef\u4ee5\u89e3\u51b3\u8be5\u95ee\u9898\u3002</p>"},{"location":"ops/faq/#pod-service","title":"Pod \u8bbf\u95ee Service \u4e0d\u901a","text":""},{"location":"ops/faq/#_4","title":"\u73b0\u8c61","text":"<p>Pod \u5185\u65e0\u6cd5\u8bbf\u95ee Service \u5bf9\u5e94\u7684\u670d\u52a1\uff0c<code>dmesg</code> \u663e\u793a\u5f02\u5e38\uff1a</p> <pre><code>netlink: Unknown conntrack attr (type=6, max=5)\nopenvswitch: netlink: Flow actions may not be safe on all matching packets.\n</code></pre> <p>\u8be5\u65e5\u5fd7\u8bf4\u660e\u5185\u6838\u5185 OVS \u7248\u672c\u8fc7\u4f4e\u4e0d\u652f\u6301\u5bf9\u5e94 NAT \u64cd\u4f5c\u3002</p>"},{"location":"ops/faq/#_5","title":"\u89e3\u51b3\u65b9\u6cd5","text":"<ol> <li>\u5347\u7ea7\u5185\u6838\u6a21\u5757\u6216\u624b\u52a8\u7f16\u8bd1 OVS \u5185\u6838\u6a21\u5757\u3002</li> <li>\u82e5\u53ea\u4f7f\u7528 Overlay \u7f51\u7edc\u53ef\u4ee5\u66f4\u6539 <code>kube-ovn-controller</code> \u542f\u52a8\u53c2\u6570\u8bbe\u7f6e <code>--enable-lb=false</code> \u5173\u95ed OVN LB \u4f7f\u7528 kube-proxy \u8fdb\u884c Service \u8f6c\u53d1\u3002</li> </ol> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"ops/from-calico/","title":"\u5378\u8f7d Calico \u5b89\u88c5 Kube-OVN","text":"<p>\u82e5 Kubernetes \u96c6\u7fa4\u5df2\u5b89\u88c5 Calico \u9700\u8981\u53d8\u66f4\u4e3a Kube-OVN \u53ef\u4ee5\u53c2\u8003\u672c\u6587\u6863\u3002</p> <p>\u672c\u6587\u4ee5 Calico v3.24.1 \u4e3a\u4f8b\uff0c\u5176\u5b83 Calico \u7248\u672c\u9700\u8981\u6839\u636e\u5b9e\u9645\u60c5\u51b5\u8fdb\u884c\u8c03\u6574\u3002</p>"},{"location":"ops/from-calico/#_1","title":"\u51c6\u5907\u5de5\u4f5c","text":"<p>\u4e3a\u4e86\u4fdd\u8bc1\u5207\u6362 CNI \u8fc7\u7a0b\u4e2d\u96c6\u7fa4\u7f51\u7edc\u4fdd\u6301\u7545\u901a\uff0cCalico ippool \u9700\u8981\u5f00\u542f nat outgoing\uff0c\u6216\u5728\u6240\u6709\u8282\u70b9\u4e0a\u5173\u95ed rp_filter\uff1a</p> <pre><code>sysctl net.ipv4.conf.all.rp_filter=0\nsysctl net.ipv4.conf.default.rp_filter=0\n# IPIP \u6a21\u5f0f\nsysctl net.ipv4.conf.tunl0.rp_filter=0\n# VXLAN \u6a21\u5f0f\nsysctl net.ipv4.conf.vxlan/calico.rp_filter=0\n# \u8def\u7531\u6a21\u5f0f\uff0ceth0 \u9700\u8981\u4fee\u6539\u4e3a\u5b9e\u9645\u4f7f\u7528\u7684\u7f51\u5361\nsysctl net.ipv4.conf.eth0.rp_filter=0\n</code></pre>"},{"location":"ops/from-calico/#kube-ovn","title":"\u90e8\u7f72 Kube-OVN","text":""},{"location":"ops/from-calico/#_2","title":"\u4e0b\u8f7d\u5b89\u88c5\u811a\u672c","text":"<pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/images/install.sh\n</code></pre>"},{"location":"ops/from-calico/#_3","title":"\u4fee\u6539\u5b89\u88c5\u811a\u672c","text":"<p>\u5c06\u5b89\u88c5\u811a\u672c\u4e2d\u91cd\u5efa Pod \u7684\u90e8\u5206\u5220\u9664\uff1a</p> <pre><code>echo \"[Step 4/6] Delete pod that not in host network mode\"\nfor ns in $(kubectl get ns --no-headers -o custom-columns=NAME:.metadata.name); do\nfor pod in $(kubectl get pod --no-headers -n \"$ns\" --field-selector spec.restartPolicy=Always -o custom-columns=NAME:.metadata.name,HOST:spec.hostNetwork | awk '{if ($2!=\"true\") print $1}'); do\nkubectl delete pod \"$pod\" -n \"$ns\" --ignore-not-found\n  done\ndone\n</code></pre> <p>\u6309\u9700\u4fee\u6539\u4ee5\u4e0b\u914d\u7f6e\uff1a</p> <pre><code>REGISTRY=\"kubeovn\"                     # \u955c\u50cf\u4ed3\u5e93\u5730\u5740\nVERSION=\"v1.13.0\"                      # \u955c\u50cf\u7248\u672c/Tag\nPOD_CIDR=\"10.16.0.0/16\"                # \u9ed8\u8ba4\u5b50\u7f51 CIDR \u4e0d\u8981\u548c SVC/NODE/JOIN CIDR \u91cd\u53e0\nSVC_CIDR=\"10.96.0.0/12\"                # \u9700\u8981\u548c apiserver \u7684 service-cluster-ip-range \u4fdd\u6301\u4e00\u81f4\nJOIN_CIDR=\"100.64.0.0/16\"              # Pod \u548c\u4e3b\u673a\u901a\u4fe1\u7f51\u7edc CIDR\uff0c\u4e0d\u8981\u548c SVC/NODE/POD CIDR \u91cd\u53e0 \nLABEL=\"node-role.kubernetes.io/master\" # \u90e8\u7f72 OVN DB \u8282\u70b9\u7684\u6807\u7b7e\nIFACE=\"\"                               # \u5bb9\u5668\u7f51\u7edc\u6240\u4f7f\u7528\u7684\u7684\u5bbf\u4e3b\u673a\u7f51\u5361\u540d\uff0c\u5982\u679c\u4e3a\u7a7a\u5219\u4f7f\u7528 Kubernetes \u4e2d\u7684 Node IP \u6240\u5728\u7f51\u5361\nTUNNEL_TYPE=\"geneve\"                   # \u96a7\u9053\u5c01\u88c5\u534f\u8bae\uff0c\u53ef\u9009 geneve, vxlan \u6216 stt\uff0cstt \u9700\u8981\u5355\u72ec\u7f16\u8bd1 ovs \u5185\u6838\u6a21\u5757\n</code></pre> <p>\u6ce8\u610f\uff1aPOD_CIDR \u53ca JOIN_CIDR \u4e0d\u53ef\u4e0e Calico ippool \u7684 CIDR \u51b2\u7a81\uff0c\u4e14 POD_CIDR \u9700\u8981\u5305\u542b\u8db3\u591f\u591a\u7684 IP \u6765\u5bb9\u7eb3\u96c6\u7fa4\u4e2d\u5df2\u6709\u7684 Pod\u3002</p>"},{"location":"ops/from-calico/#_4","title":"\u6267\u884c\u5b89\u88c5\u811a\u672c","text":"<pre><code>bash install.sh\n</code></pre>"},{"location":"ops/from-calico/#_5","title":"\u9010\u4e2a\u8282\u70b9\u8fc1\u79fb","text":"<p>\u6309\u7167\u4ee5\u4e0b\u65b9\u6cd5\u4e3a\u6bcf\u4e2a\u8282\u70b9\u9010\u4e2a\u8fdb\u884c\u8fc1\u79fb\u3002 \u6ce8\u610f\uff1a\u547d\u4ee4\u4e2d\u7684 \\&lt;NODE&gt; \u9700\u8981\u66ff\u6362\u4e3a\u8282\u70b9\u540d\u79f0\u3002</p>"},{"location":"ops/from-calico/#_6","title":"\u9a71\u9010\u8282\u70b9","text":"<pre><code>kubectl drain --ignore-daemonsets &lt;NODE&gt;\n</code></pre> <p>\u82e5\u6b64\u547d\u4ee4\u4e00\u76f4\u7b49\u5f85 Pod \u88ab\u9a71\u9010\uff0c\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\u5f3a\u5236\u5220\u9664\u88ab\u9a71\u9010\u7684 Pod\uff1a</p> <pre><code>kubectl get pod -A --field-selector=spec.nodeName=&lt;NODE&gt; --no-headers | \\\nawk '$4==\"Terminating\" {print $1\" \"$2}' | \\\nwhile read s; do kubectl delete pod --force -n $s; done\n</code></pre>"},{"location":"ops/from-calico/#_7","title":"\u91cd\u542f\u8282\u70b9","text":"<p>\u5728\u8282\u70b9\u4e2d\u6267\u884c\uff1a</p> <pre><code>shutdown -r 0\n</code></pre>"},{"location":"ops/from-calico/#_8","title":"\u6062\u590d\u8282\u70b9","text":"<pre><code>kubectl uncordon &lt;NODE&gt;\n</code></pre>"},{"location":"ops/from-calico/#calico","title":"\u5378\u8f7d Calico","text":""},{"location":"ops/from-calico/#k8s","title":"\u5220\u9664 k8s \u8d44\u6e90","text":"<pre><code>kubectl -n kube-system delete deploy calico-kube-controllers\nkubectl -n kube-system delete ds calico-node\nkubectl -n kube-system delete cm calico-config\n# \u5220\u9664 CRD \u53ca\u76f8\u5173\u8d44\u6e90\nkubectl get crd -o jsonpath='{range .items[*]}{.metadata.name}{\"\\n\"}{end}' | while read crd; do\nif ! echo $crd | grep '.crd.projectcalico.org$' &gt;/dev/null; then\ncontinue\nfi\n\nfor name in $(kubectl get $crd -o jsonpath='{.items[*].metadata.name}'); do\nkubectl delete $crd $name\ndone\nkubectl delete crd $crd\ndone\n# \u5176\u5b83\u8d44\u6e90\nkubectl delete --ignore-not-found clusterrolebinding calico-node calico-kube-controllers\nkubectl delete --ignore-not-found clusterrole calico-node calico-kube-controllers\nkubectl delete --ignore-not-found sa -n kube-system calico-kube-controllers calico-node\nkubectl delete --ignore-not-found pdb -n kube-system calico-kube-controllers\n</code></pre>"},{"location":"ops/from-calico/#_9","title":"\u6e05\u7406\u8282\u70b9\u6587\u4ef6","text":"<p>\u5728\u6bcf\u4e2a\u8282\u70b9\u4e2d\u6267\u884c\uff1a</p> <pre><code>rm -f /etc/cni/net.d/10-calico.conflist /etc/cni/net.d/calico-kubeconfig\nrm -f /opt/cni/bin/calico /opt/cni/bin/calico-ipam\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"ops/kubectl-ko/","title":"kubectl \u63d2\u4ef6\u4f7f\u7528","text":"<p>\u4e3a\u4e86\u65b9\u4fbf\u65e5\u5e38\u7684\u8fd0\u7ef4\u64cd\u4f5c\uff0cKube-OVN \u63d0\u4f9b\u4e86 kubectl \u63d2\u4ef6\u5de5\u5177\uff0c\u7f51\u7edc\u7ba1\u7406\u5458 \u53ef\u4ee5\u901a\u8fc7\u8be5\u547d\u4ee4\u8fdb\u884c\u65e5\u5e38\u64cd\u4f5c\uff0c\u4f8b\u5982\uff1a\u67e5\u770b OVN \u6570\u636e\u5e93\u4fe1\u606f\u548c\u72b6\u6001\uff0cOVN \u6570\u636e\u5e93 \u5907\u4efd\u548c\u6062\u590d\uff0cOVS \u76f8\u5173\u4fe1\u606f\u67e5\u770b\uff0ctcpdump \u7279\u5b9a\u5bb9\u5668\uff0c\u7279\u5b9a\u94fe\u8def\u903b\u8f91\u62d3\u6251\u5c55\u793a\uff0c \u7f51\u7edc\u95ee\u9898\u8bca\u65ad\u548c\u6027\u80fd\u4f18\u5316\u3002</p>"},{"location":"ops/kubectl-ko/#_1","title":"\u63d2\u4ef6\u5b89\u88c5","text":"<p>Kube-OVN \u5b89\u88c5\u65f6\u9ed8\u8ba4\u4f1a\u90e8\u7f72\u63d2\u4ef6\u5230\u6bcf\u4e2a\u8282\u70b9\uff0c\u82e5\u6267\u884c kubectl \u7684\u673a\u5668\u4e0d\u5728\u96c6\u7fa4\u5185\uff0c \u6216\u9700\u8981\u91cd\u88c5\u63d2\u4ef6\uff0c\u53ef\u53c2\u8003\u4e0b\u9762\u7684\u6b65\u9aa4\uff1a</p> <p>\u4e0b\u8f7d <code>kubectl-ko</code> \u6587\u4ef6\uff1a</p> <pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/images/kubectl-ko\n</code></pre> <p>\u5c06\u8be5\u6587\u4ef6\u79fb\u52a8\u81f3 <code>$PATH</code> \u76ee\u5f55\u4e0b\uff1a</p> <pre><code>mv kubectl-ko /usr/local/bin/kubectl-ko\n</code></pre> <p>\u589e\u52a0\u53ef\u6267\u884c\u6743\u9650\uff1a</p> <pre><code>chmod +x /usr/local/bin/kubectl-ko\n</code></pre> <p>\u68c0\u67e5\u63d2\u4ef6\u662f\u5426\u53ef\u4ee5\u6b63\u5e38\u4f7f\u7528\uff1a</p> <pre><code># kubectl plugin list\nThe following compatible plugins are available:\n\n/usr/local/bin/kubectl-ko\n</code></pre>"},{"location":"ops/kubectl-ko/#_2","title":"\u63d2\u4ef6\u4f7f\u7528","text":"<p>\u8fd0\u884c <code>kubectl ko</code> \u4f1a\u5c55\u793a\u8be5\u63d2\u4ef6\u6240\u6709\u53ef\u7528\u7684\u547d\u4ee4\u548c\u7528\u6cd5\u63cf\u8ff0\uff0c\u5982\u4e0b\u6240\u793a\uff1a</p> <pre><code># kubectl ko\nkubectl ko {subcommand} [option...]\nAvailable Subcommands:\n  [nb|sb] [status|kick|backup|dbstatus|restore]     ovn-db operations show cluster status, kick stale server, backup database, get db consistency status or restore ovn nb db when met 'inconsistent data' error\n  nbctl [ovn-nbctl options ...]    invoke ovn-nbctl\n  sbctl [ovn-sbctl options ...]    invoke ovn-sbctl\n  vsctl {nodeName} [ovs-vsctl options ...]   invoke ovs-vsctl on the specified node\n  ofctl {nodeName} [ovs-ofctl options ...]   invoke ovs-ofctl on the specified node\n  dpctl {nodeName} [ovs-dpctl options ...]   invoke ovs-dpctl on the specified node\n  appctl {nodeName} [ovs-appctl options ...]   invoke ovs-appctl on the specified node\n  tcpdump {namespace/podname} [tcpdump options ...]     capture pod traffic\n  trace {namespace/podname} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp/udp port]    trace ICMP/TCP/UDP\n  trace {namespace/podname} {target ip address} [target mac address] arp {request|reply}                     trace ARP request/reply\n  trace {node//nodename} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp/udp port]       trace ICMP/TCP/UDP\n  trace {node//nodename} {target ip address} [target mac address] arp {request|reply}                        trace ARP request/reply\n  echo \"  diagnose {all|node|subnet|IPPorts} [nodename|subnetName|{proto1}-{IP1}-{Port1},{proto2}-{IP2}-{Port2}]    diagnose connectivity of all nodes or a specific node or specify subnet's ds pod or IPPorts like 'tcp-172.18.0.2-53,udp-172.18.0.3-53'\"\ntuning {install-fastpath|local-install-fastpath|remove-fastpath|install-stt|local-install-stt|remove-stt} {centos7|centos8}} [kernel-devel-version]  deploy  kernel optimisation components to the system\n  reload    restart all kube-ovn components\n  log {kube-ovn|ovn|ovs|linux|all}    save log to ./kubectl-ko-log/\n  perf [image] performance test default image is kubeovn/test:v1.12.0  </code></pre> <p>\u4e0b\u9762\u5c06\u4ecb\u7ecd\u6bcf\u4e2a\u547d\u4ee4\u7684\u5177\u4f53\u529f\u80fd\u548c\u4f7f\u7528\u3002</p>"},{"location":"ops/kubectl-ko/#nb-sb-status-kick-backup-dbstatus-restore","title":"[nb | sb] [status | kick | backup | dbstatus | restore]","text":"<p>\u8be5\u5b50\u547d\u4ee4\u4e3b\u8981\u5bf9 OVN \u5317\u5411\u6216\u5357\u5411\u6570\u636e\u5e93\u8fdb\u884c\u64cd\u4f5c\uff0c\u5305\u62ec\u6570\u636e\u5e93\u96c6\u7fa4\u72b6\u6001\u67e5\u770b\uff0c\u6570\u636e\u5e93\u8282\u70b9\u4e0b\u7ebf\uff0c \u6570\u636e\u5e93\u5907\u4efd\uff0c\u6570\u636e\u5e93\u5b58\u50a8\u72b6\u6001\u67e5\u770b\u548c\u6570\u636e\u5e93\u4fee\u590d\u3002</p>"},{"location":"ops/kubectl-ko/#_3","title":"\u6570\u636e\u5e93\u96c6\u7fa4\u72b6\u6001\u67e5\u770b","text":"<p>\u8be5\u547d\u4ee4\u4f1a\u5728\u5bf9\u5e94 OVN \u6570\u636e\u5e93\u7684 leader \u8282\u70b9\u6267\u884c <code>ovs-appctl cluster/status</code> \u5c55\u793a\u96c6\u7fa4\u72b6\u6001:</p> <pre><code># kubectl ko nb status\n306b\nName: OVN_Northbound\nCluster ID: 9a87 (9a872522-3e7d-47ca-83a3-d74333e1a7ca)\nServer ID: 306b (306b256b-b5e1-4eb0-be91-4ca96adf6bad)\nAddress: tcp:[172.18.0.2]:6643\nStatus: cluster member\nRole: leader\nTerm: 1\nLeader: self\nVote: self\n\nLast Election started 280309 ms ago, reason: timeout\nLast Election won: 280309 ms ago\nElection timer: 5000\nLog: [139, 139]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: &lt;-8723 -&gt;8723 &lt;-85d6 -&gt;85d6\nDisconnections: 0\nServers:\n    85d6 (85d6 at tcp:[172.18.0.4]:6643) next_index=139 match_index=138 last msg 763 ms ago\n    8723 (8723 at tcp:[172.18.0.3]:6643) next_index=139 match_index=138 last msg 763 ms ago\n    306b (306b at tcp:[172.18.0.2]:6643) (self) next_index=2 match_index=138\nstatus: ok\n</code></pre> <p>\u82e5 <code>Server</code> \u4e0b\u7684 <code>match_index</code> \u51fa\u73b0\u8f83\u5927\u5dee\u522b\uff0c\u4e14 <code>last msg</code> \u65f6\u95f4\u8f83\u957f\u5219\u5bf9\u5e94 Server \u53ef\u80fd\u957f\u65f6\u95f4\u6ca1\u6709\u54cd\u5e94\uff0c \u9700\u8981\u8fdb\u4e00\u6b65\u67e5\u770b\u3002</p>"},{"location":"ops/kubectl-ko/#_4","title":"\u6570\u636e\u5e93\u8282\u70b9\u4e0b\u7ebf","text":"<p>\u8be5\u547d\u4ee4\u4f1a\u5c06\u67d0\u4e2a\u8282\u70b9\u4ece OVN \u6570\u636e\u5e93\u4e2d\u79fb\u9664\uff0c\u5728\u8282\u70b9\u4e0b\u7ebf\u6216\u66f4\u6362\u8282\u70b9\u65f6\u9700\u8981\u7528\u5230\u3002 \u4e0b\u9762\u5c06\u4ee5\u4e0a\u4e00\u6761\u547d\u4ee4\u6240\u67e5\u770b\u5230\u7684\u96c6\u7fa4\u72b6\u6001\u4e3a\u4f8b\uff0c\u4e0b\u7ebf <code>172.18.0.3</code> \u8282\u70b9:</p> <pre><code># kubectl ko nb kick 8723\nstarted removal\n</code></pre> <p>\u518d\u6b21\u67e5\u770b\u6570\u636e\u5e93\u96c6\u7fa4\u72b6\u6001\u786e\u8ba4\u8282\u70b9\u5df2\u79fb\u9664\uff1a</p> <pre><code># kubectl ko nb status\n306b\nName: OVN_Northbound\nCluster ID: 9a87 (9a872522-3e7d-47ca-83a3-d74333e1a7ca)\nServer ID: 306b (306b256b-b5e1-4eb0-be91-4ca96adf6bad)\nAddress: tcp:[172.18.0.2]:6643\nStatus: cluster member\nRole: leader\nTerm: 1\nLeader: self\nVote: self\n\nLast Election started 324356 ms ago, reason: timeout\nLast Election won: 324356 ms ago\nElection timer: 5000\nLog: [140, 140]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: &lt;-85d6 -&gt;85d6\nDisconnections: 2\nServers:\n    85d6 (85d6 at tcp:[172.18.0.4]:6643) next_index=140 match_index=139 last msg 848 ms ago\n    306b (306b at tcp:[172.18.0.2]:6643) (self) next_index=2 match_index=139\nstatus: ok\n</code></pre>"},{"location":"ops/kubectl-ko/#_5","title":"\u6570\u636e\u5e93\u5907\u4efd","text":"<p>\u8be5\u5b50\u547d\u4ee4\u4f1a\u5907\u4efd\u5f53\u524d OVN \u6570\u636e\u5e93\u81f3\u672c\u5730\uff0c\u53ef\u7528\u4e8e\u707e\u5907\u548c\u6062\u590d\uff1a</p> <pre><code># kubectl ko nb backup\ntar: Removing leading `/' from member names\nbackup ovn-nb db to /root/ovnnb_db.060223191654183154.backup\n</code></pre>"},{"location":"ops/kubectl-ko/#_6","title":"\u6570\u636e\u5e93\u5b58\u50a8\u72b6\u6001\u67e5\u770b","text":"<p>\u8be5\u547d\u4ee4\u7528\u6765\u67e5\u770b\u6570\u636e\u5e93\u6587\u4ef6\u662f\u5426\u5b58\u5728\u635f\u574f\uff1a</p> <pre><code># kubectl ko nb dbstatus\nstatus: ok\n</code></pre> <p>\u82e5\u5f02\u5e38\u5219\u663e\u793a <code>inconsistent data</code> \u9700\u8981\u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u8fdb\u884c\u4fee\u590d\u3002</p>"},{"location":"ops/kubectl-ko/#_7","title":"\u6570\u636e\u5e93\u4fee\u590d","text":"<p>\u82e5\u6570\u636e\u5e93\u72b6\u6001\u8fdb\u5165 <code>inconsistent data</code> \u53ef\u4f7f\u7528\u8be5\u547d\u4ee4\u8fdb\u884c\u4fee\u590d\uff1a</p> <pre><code># kubectl ko nb restore\ndeployment.apps/ovn-central scaled\novn-central original replicas is 3\nfirst nodeIP is 172.18.0.5\novs-ovn pod on node 172.18.0.5 is ovs-ovn-8jxv9\novs-ovn pod on node 172.18.0.3 is ovs-ovn-sjzb6\novs-ovn pod on node 172.18.0.4 is ovs-ovn-t87zk\nbackup nb db file\nrestore nb db file, operate in pod ovs-ovn-8jxv9\ndeployment.apps/ovn-central scaled\nfinish restore nb db file and ovn-central replicas\nrecreate ovs-ovn pods\npod \"ovs-ovn-8jxv9\" deleted\npod \"ovs-ovn-sjzb6\" deleted\npod \"ovs-ovn-t87zk\" deleted\n</code></pre>"},{"location":"ops/kubectl-ko/#nbctl-sbctl-options","title":"[nbctl | sbctl] [options ...]","text":"<p>\u8be5\u5b50\u547d\u4ee4\u4f1a\u76f4\u63a5\u8fdb\u5165 OVN \u5317\u5411\u6570\u636e\u5e93\u6216\u5357\u5411\u6570\u636e\u5e93 \u7684 leader \u8282\u70b9\u5206\u522b\u6267\u884c <code>ovn-nbctl</code> \u548c <code>ovn-sbctl</code> \u547d\u4ee4\u3002 \u66f4\u591a\u8be5\u547d\u4ee4\u7684\u8be6\u7ec6\u7528\u6cd5\u8bf7\u67e5\u8be2\u4e0a\u6e38 OVN \u7684\u5b98\u65b9\u6587\u6863 ovn-nbctl(8) \u548c ovn-sbctl(8)\u3002</p> <pre><code># kubectl ko nbctl show\nswitch c7cd17e8-ceee-4a91-9bb3-e5a313fe1ece (snat)\nport snat-ovn-cluster\n        type: router\n        router-port: ovn-cluster-snat\nswitch 20e0c6d0-023a-4756-aec5-200e0c60f95d (join)\nport node-liumengxin-ovn3-192.168.137.178\n        addresses: [\"00:00:00:64:FF:A8 100.64.0.4\"]\nport node-liumengxin-ovn1-192.168.137.176\n        addresses: [\"00:00:00:AF:98:62 100.64.0.2\"]\nport node-liumengxin-ovn2-192.168.137.177\n        addresses: [\"00:00:00:D9:58:B8 100.64.0.3\"]\nport join-ovn-cluster\n        type: router\n        router-port: ovn-cluster-join\nswitch 0191705c-f827-427b-9de3-3c3b7d971ba5 (central)\nport central-ovn-cluster\n        type: router\n        router-port: ovn-cluster-central\nswitch 2a45ff05-388d-4f85-9daf-e6fccd5833dc (ovn-default)\nport alertmanager-main-0.monitoring\n        addresses: [\"00:00:00:6C:DF:A3 10.16.0.19\"]\nport kube-state-metrics-5d6885d89-4nf8h.monitoring\n        addresses: [\"00:00:00:6F:02:1C 10.16.0.15\"]\nport fake-kubelet-67c55dfd89-pv86k.kube-system\n        addresses: [\"00:00:00:5C:12:E8 10.16.19.177\"]\nport ovn-default-ovn-cluster\n        type: router\n        router-port: ovn-cluster-ovn-default\nrouter 212f73dd-d63d-4d72-864b-a537e9afbee1 (ovn-cluster)\nport ovn-cluster-snat\n        mac: \"00:00:00:7A:82:8F\"\nnetworks: [\"172.22.0.1/16\"]\nport ovn-cluster-join\n        mac: \"00:00:00:F8:18:5A\"\nnetworks: [\"100.64.0.1/16\"]\nport ovn-cluster-central\n        mac: \"00:00:00:4D:8C:F5\"\nnetworks: [\"192.101.0.1/16\"]\nport ovn-cluster-ovn-default\n        mac: \"00:00:00:A3:F8:18\"\nnetworks: [\"10.16.0.1/16\"]\n</code></pre>"},{"location":"ops/kubectl-ko/#vsctl-nodename-options","title":"vsctl {nodeName} [options ...]","text":"<p>\u8be5\u547d\u4ee4\u4f1a\u8fdb\u5165\u5bf9\u5e94 <code>nodeName</code> \u4e0a\u7684 <code>ovs-ovn</code> \u5bb9\u5668\uff0c\u5e76\u6267\u884c\u76f8\u5e94\u7684 <code>ovs-vsctl</code> \u547d\u4ee4\uff0c\u67e5\u8be2\u5e76\u914d\u7f6e <code>vswitchd</code>\u3002 \u66f4\u591a\u8be5\u547d\u4ee4\u7684\u8be6\u7ec6\u7528\u6cd5\u8bf7\u67e5\u8be2\u4e0a\u6e38 OVS \u7684\u5b98\u65b9\u6587\u6863 ovs-vsctl(8)\u3002</p> <pre><code># kubectl ko vsctl kube-ovn-01 show\n0d4c4675-c9cc-440a-8c1a-878e17f81b88\n    Bridge br-int\n        fail_mode: secure\n        datapath_type: system\n        Port a2c1a8a8b83a_h\n            Interface a2c1a8a8b83a_h\n        Port \"4fa5c4cbb1a5_h\"\nInterface \"4fa5c4cbb1a5_h\"\nPort ovn-eef07d-0\n            Interface ovn-eef07d-0\n                type: stt\n                options: {csum=\"true\", key=flow, remote_ip=\"192.168.137.178\"}\nPort ovn0\n            Interface ovn0\n                type: internal\n        Port mirror0\n            Interface mirror0\n                type: internal\n        Port ovn-efa253-0\n            Interface ovn-efa253-0\n                type: stt\n                options: {csum=\"true\", key=flow, remote_ip=\"192.168.137.177\"}\nPort br-int\n            Interface br-int\n                type: internal\n    ovs_version: \"2.17.2\"\n</code></pre>"},{"location":"ops/kubectl-ko/#ofctl-nodename-options","title":"ofctl {nodeName} [options ...]","text":"<p>\u8be5\u547d\u4ee4\u4f1a\u8fdb\u5165\u5bf9\u5e94 <code>nodeName</code> \u4e0a\u7684 <code>ovs-ovn</code> \u5bb9\u5668\uff0c\u5e76\u6267\u884c\u76f8\u5e94\u7684 <code>ovs-ofctl</code> \u547d\u4ee4\uff0c\u67e5\u8be2\u6216\u7ba1\u7406 OpenFlow\u3002 \u66f4\u591a\u8be5\u547d\u4ee4\u7684\u8be6\u7ec6\u7528\u6cd5\u8bf7\u67e5\u8be2\u4e0a\u6e38 OVS \u7684\u5b98\u65b9\u6587\u6863 ovs-ofctl(8)\u3002</p> <pre><code># kubectl ko ofctl kube-ovn-01 dump-flows br-int\nNXST_FLOW reply (xid=0x4): flags=[more]\ncookie=0xcf3429e6, duration=671791.432s, table=0, n_packets=0, n_bytes=0, idle_age=65534, hard_age=65534, priority=100,in_port=2 actions=load:0x4-&gt;NXM_NX_REG13[],load:0x9-&gt;NXM_NX_REG11[],load:0xb-&gt;NXM_NX_REG12[],load:0x4-&gt;OXM_OF_METADATA[],load:0x1-&gt;NXM_NX_REG14[],resubmit(,8)\ncookie=0xc91413c6, duration=671791.431s, table=0, n_packets=907489, n_bytes=99978275, idle_age=0, hard_age=65534, priority=100,in_port=7 actions=load:0x1-&gt;NXM_NX_REG13[],load:0x9-&gt;NXM_NX_REG11[],load:0xb-&gt;NXM_NX_REG12[],load:0x4-&gt;OXM_OF_METADATA[],load:0x4-&gt;NXM_NX_REG14[],resubmit(,8)\ncookie=0xf180459, duration=671791.431s, table=0, n_packets=17348582, n_bytes=2667811214, idle_age=0, hard_age=65534, priority=100,in_port=6317 actions=load:0xa-&gt;NXM_NX_REG13[],load:0x9-&gt;NXM_NX_REG11[],load:0xb-&gt;NXM_NX_REG12[],load:0x4-&gt;OXM_OF_METADATA[],load:0x9-&gt;NXM_NX_REG14[],resubmit(,8)\ncookie=0x7806dd90, duration=671791.431s, table=0, n_packets=3235428, n_bytes=833821312, idle_age=0, hard_age=65534, priority=100,in_port=1 actions=load:0xd-&gt;NXM_NX_REG13[],load:0x9-&gt;NXM_NX_REG11[],load:0xb-&gt;NXM_NX_REG12[],load:0x4-&gt;OXM_OF_METADATA[],load:0x3-&gt;NXM_NX_REG14[],resubmit(,8)\n...\n</code></pre>"},{"location":"ops/kubectl-ko/#dpctl-nodename-options","title":"dpctl {nodeName} [options ...]","text":"<p>\u8be5\u547d\u4ee4\u4f1a\u8fdb\u5165\u5bf9\u5e94 <code>nodeName</code> \u4e0a\u7684 <code>ovs-ovn</code> \u5bb9\u5668\uff0c\u5e76\u6267\u884c\u76f8\u5e94\u7684 <code>ovs-dpctl</code> \u547d\u4ee4\uff0c\u67e5\u8be2\u6216\u7ba1\u7406 OVS datapath\u3002 \u66f4\u591a\u8be5\u547d\u4ee4\u7684\u8be6\u7ec6\u7528\u6cd5\u8bf7\u67e5\u8be2\u4e0a\u6e38 OVS \u7684\u5b98\u65b9\u6587\u6863 ovs-dpctl(8)\u3002</p> <pre><code># kubectl ko dpctl kube-ovn-01 show\nsystem@ovs-system:\n  lookups: hit:350805055 missed:21983648 lost:73\n  flows: 105\nmasks: hit:1970748791 total:22 hit/pkt:5.29\n  port 0: ovs-system (internal)\nport 1: ovn0 (internal)\nport 2: mirror0 (internal)\nport 3: br-int (internal)\nport 4: stt_sys_7471 (stt: packet_type=ptap)\nport 5: eeb4d9e51b5d_h\n  port 6: a2c1a8a8b83a_h\n  port 7: 4fa5c4cbb1a5_h\n</code></pre>"},{"location":"ops/kubectl-ko/#appctl-nodename-options","title":"appctl {nodeName} [options ...]","text":"<p>\u8be5\u547d\u4ee4\u4f1a\u8fdb\u5165\u5bf9\u5e94 <code>nodeName</code> \u4e0a\u7684 <code>ovs-ovn</code> \u5bb9\u5668\uff0c\u5e76\u6267\u884c\u76f8\u5e94\u7684 <code>ovs-appctl</code> \u547d\u4ee4\uff0c\u6765\u64cd\u4f5c\u76f8\u5173 daemon \u8fdb\u7a0b\u3002 \u66f4\u591a\u8be5\u547d\u4ee4\u7684\u8be6\u7ec6\u7528\u6cd5\u8bf7\u67e5\u8be2\u4e0a\u6e38 OVS \u7684\u5b98\u65b9\u6587\u6863 ovs-appctl(8)\u3002</p> <pre><code># kubectl ko appctl kube-ovn-01 vlog/list\nconsole    syslog    file\n                 -------    ------    ------\nbacktrace          OFF        ERR       INFO\nbfd                OFF        ERR       INFO\nbond               OFF        ERR       INFO\nbridge             OFF        ERR       INFO\nbundle             OFF        ERR       INFO\nbundles            OFF        ERR       INFO\n...\n</code></pre>"},{"location":"ops/kubectl-ko/#tcpdump-namespacepodname-tcpdump-options","title":"tcpdump {namespace/podname} [tcpdump options ...]","text":"<p>\u8be5\u547d\u4ee4\u4f1a\u8fdb\u5165 <code>namespace/podname</code> \u6240\u5728\u673a\u5668\u7684 <code>kube-ovn-cni</code> \u5bb9\u5668\uff0c\u5e76\u6267\u884c <code>tcpdump</code> \u6293\u53d6\u5bf9\u5e94\u5bb9\u5668 veth \u7f51\u5361 \u7aef\u7684\u6d41\u91cf\uff0c\u53ef\u4ee5\u65b9\u4fbf\u6392\u67e5\u7f51\u7edc\u76f8\u5173\u95ee\u9898\uff0c\u5982\u4e0b\u6240\u793a\uff1a</p> <pre><code># kubectl ko tcpdump default/ds1-l6n7p icmp\n+ kubectl exec -it kube-ovn-cni-wlg4s -n kube-ovn -- tcpdump -nn -i d7176fe7b4e0_h icmp\ntcpdump: verbose output suppressed, use -v or -vv for full protocol decode\nlistening on d7176fe7b4e0_h, link-type EN10MB (Ethernet), capture size 262144 bytes\n06:52:36.619688 IP 100.64.0.3 &gt; 10.16.0.4: ICMP echo request, id 2, seq 1, length 64\n06:52:36.619746 IP 10.16.0.4 &gt; 100.64.0.3: ICMP echo reply, id 2, seq 1, length 64\n06:52:37.619588 IP 100.64.0.3 &gt; 10.16.0.4: ICMP echo request, id 2, seq 2, length 64\n06:52:37.619630 IP 10.16.0.4 &gt; 100.64.0.3: ICMP echo reply, id 2, seq 2, length 64\n06:52:38.619933 IP 100.64.0.3 &gt; 10.16.0.4: ICMP echo request, id 2, seq 3, length 64\n06:52:38.619973 IP 10.16.0.4 &gt; 100.64.0.3: ICMP echo reply, id 2, seq 3, length 64\n</code></pre>"},{"location":"ops/kubectl-ko/#trace-arguments","title":"trace [arguments ...]","text":"<p>\u8be5\u547d\u4ee4\u5c06\u4f1a\u6253\u5370 Pod \u6216\u8282\u70b9\u901a\u8fc7\u7279\u5b9a\u534f\u8bae\u8bbf\u95ee\u67d0\u5730\u5740\u65f6\u5bf9\u5e94\u7684 OVN \u903b\u8f91\u6d41\u8868\u548c\u6700\u7ec8\u7684 Openflow \u6d41\u8868\uff0c \u65b9\u4fbf\u5f00\u53d1\u6216\u8fd0\u7ef4\u65f6\u5b9a\u4f4d\u6d41\u8868\u76f8\u5173\u95ee\u9898\u3002</p> <p>\u652f\u6301\u7684\u547d\u4ee4\uff1a</p> <pre><code>kubectl ko trace {namespace/podname} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp/udp port]\nkubectl ko trace {namespace/podname} {target ip address} [target mac address] arp {request|reply}\nkubectl ko trace {node//nodename} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp/udp port]\nkubectl ko trace {node//nodename} {target ip address} [target mac address] arp {request|reply}\n</code></pre> <p>\u793a\u4f8b\uff1a</p> <pre><code># kubectl ko trace default/ds1-l6n7p 8.8.8.8 icmp\n+ kubectl exec ovn-central-5bc494cb5-np9hm -n kube-ovn -- ovn-trace --ct=new ovn-default 'inport == \"ds1-l6n7p.default\" &amp;&amp; ip.ttl == 64 &amp;&amp; icmp &amp;&amp; eth.src == 0a:00:00:10:00:05 &amp;&amp; ip4.src == 10.16.0.4 &amp;&amp; eth.dst == 00:00:00:B8:CA:43 &amp;&amp; ip4.dst == 8.8.8.8'\n# icmp,reg14=0xf,vlan_tci=0x0000,dl_src=0a:00:00:10:00:05,dl_dst=00:00:00:b8:ca:43,nw_src=10.16.0.4,nw_dst=8.8.8.8,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=0,icmp_code=0\n\ningress(dp=\"ovn-default\", inport=\"ds1-l6n7p.default\")\n-----------------------------------------------------\n 0. ls_in_port_sec_l2 (ovn-northd.c:4143): inport == \"ds1-l6n7p.default\" &amp;&amp; eth.src == {0a:00:00:10:00:05}, priority 50, uuid 39453393\nnext;\n1. ls_in_port_sec_ip (ovn-northd.c:2898): inport == \"ds1-l6n7p.default\" &amp;&amp; eth.src == 0a:00:00:10:00:05 &amp;&amp; ip4.src == {10.16.0.4}, priority 90, uuid 81bcd485\n    next;\n3. ls_in_pre_acl (ovn-northd.c:3269): ip, priority 100, uuid 7b4f4971\n    reg0[0] = 1;\nnext;\n5. ls_in_pre_stateful (ovn-northd.c:3396): reg0[0] == 1, priority 100, uuid 36cdd577\n    ct_next;\n\nct_next(ct_state=new|trk)\n-------------------------\n 6. ls_in_acl (ovn-northd.c:3759): ip &amp;&amp; (!ct.est || (ct.est &amp;&amp; ct_label.blocked == 1)), priority 1, uuid 7608af5b\n    reg0[1] = 1;\nnext;\n10. ls_in_stateful (ovn-northd.c:3995): reg0[1] == 1, priority 100, uuid 2aba1b90\n    ct_commit(ct_label=0/0x1);\nnext;\n16. ls_in_l2_lkup (ovn-northd.c:4470): eth.dst == 00:00:00:b8:ca:43, priority 50, uuid 5c9c3c9f\n    outport = \"ovn-default-ovn-cluster\";\noutput;\n\n...\n</code></pre> <p>\u82e5 trace \u5bf9\u8c61\u4e3a\u8fd0\u884c\u4e8e Underlay \u7f51\u7edc\u4e0b\u7684\u865a\u62df\u673a\uff0c\u9700\u8981\u6dfb\u52a0\u989d\u5916\u53c2\u6570\u6765\u6307\u5b9a\u76ee\u7684 Mac \u5730\u5740\uff1a</p> <pre><code>kubectl ko trace default/virt-handler-7lvml 8.8.8.8 82:7c:9f:83:8c:01 icmp\n</code></pre>"},{"location":"ops/kubectl-ko/#diagnose-allnodesubnetipports-nodenamesubnetnameproto1-ip1-port1proto2-ip2-port2","title":"diagnose {all|node|subnet|IPPorts} [nodename|subnetName|{proto1}-{IP1}-{Port1},{proto2}-{IP2}-{Port2}]","text":"<p>\u8bca\u65ad\u96c6\u7fa4\u7f51\u7edc\u7ec4\u4ef6\u72b6\u6001\uff0c\u5e76\u53bb\u5bf9\u5e94\u8282\u70b9\u7684 <code>kube-ovn-pinger</code> \u68c0\u6d4b\u5f53\u524d\u8282\u70b9\u5230\u5176\u4ed6\u8282\u70b9\u548c\u5173\u952e\u670d\u52a1\u7684\u8fde\u901a\u6027\u548c\u7f51\u7edc\u5ef6\u8fdf\uff1a</p> <pre><code># kubectl ko diagnose all\nswitch c7cd17e8-ceee-4a91-9bb3-e5a313fe1ece (snat)\nport snat-ovn-cluster\n        type: router\n        router-port: ovn-cluster-snat\nswitch 20e0c6d0-023a-4756-aec5-200e0c60f95d (join)\nport node-liumengxin-ovn3-192.168.137.178\n        addresses: [\"00:00:00:64:FF:A8 100.64.0.4\"]\nport node-liumengxin-ovn1-192.168.137.176\n        addresses: [\"00:00:00:AF:98:62 100.64.0.2\"]\nport join-ovn-cluster\n        type: router\n        router-port: ovn-cluster-join\nswitch 0191705c-f827-427b-9de3-3c3b7d971ba5 (central)\nport central-ovn-cluster\n        type: router\n        router-port: ovn-cluster-central\nswitch 2a45ff05-388d-4f85-9daf-e6fccd5833dc (ovn-default)\nport ovn-default-ovn-cluster\n        type: router\n        router-port: ovn-cluster-ovn-default\n    port prometheus-k8s-1.monitoring\n        addresses: [\"00:00:00:AA:37:DF 10.16.0.23\"]\nrouter 212f73dd-d63d-4d72-864b-a537e9afbee1 (ovn-cluster)\nport ovn-cluster-snat\n        mac: \"00:00:00:7A:82:8F\"\nnetworks: [\"172.22.0.1/16\"]\nport ovn-cluster-join\n        mac: \"00:00:00:F8:18:5A\"\nnetworks: [\"100.64.0.1/16\"]\nport ovn-cluster-central\n        mac: \"00:00:00:4D:8C:F5\"\nnetworks: [\"192.101.0.1/16\"]\nport ovn-cluster-ovn-default\n        mac: \"00:00:00:A3:F8:18\"\nnetworks: [\"10.16.0.1/16\"]\nRouting Policies\n     31000                            ip4.dst == 10.16.0.0/16           allow\n     31000                           ip4.dst == 100.64.0.0/16           allow\n     30000                         ip4.dst == 192.168.137.177         reroute                100.64.0.3\n     30000                         ip4.dst == 192.168.137.178         reroute                100.64.0.4\n     29000                 ip4.src == $ovn.default.fake.6_ip4         reroute               100.64.0.22\n     29000                 ip4.src == $ovn.default.fake.7_ip4         reroute               100.64.0.21\n     29000                 ip4.src == $ovn.default.fake.8_ip4         reroute               100.64.0.23\n     29000 ip4.src == $ovn.default.liumengxin.ovn3.192.168.137.178_ip4         reroute                100.64.0.4\n     20000 ip4.src == $ovn.default.liumengxin.ovn1.192.168.137.176_ip4 &amp;&amp; ip4.dst != $ovn.cluster.overlay.subnets.IPv4         reroute                100.64.0.2\n     20000 ip4.src == $ovn.default.liumengxin.ovn2.192.168.137.177_ip4 &amp;&amp; ip4.dst != $ovn.cluster.overlay.subnets.IPv4         reroute                100.64.0.3\n     20000 ip4.src == $ovn.default.liumengxin.ovn3.192.168.137.178_ip4 &amp;&amp; ip4.dst != $ovn.cluster.overlay.subnets.IPv4         reroute                100.64.0.4\nIPv4 Routes\nRoute Table &lt;main&gt;:\n                0.0.0.0/0                100.64.0.1 dst-ip\nUUID                                    LB                  PROTO      VIP                     IPs\ne9bcfd9d-793e-4431-9073-6dec96b75d71    cluster-tcp-load    tcp        10.100.209.132:10660    192.168.137.176:10660\n                                                            tcp        10.101.239.192:6641     192.168.137.177:6641\n                                                            tcp        10.101.240.101:3000     10.16.0.7:3000\n                                                            tcp        10.103.184.186:6642     192.168.137.177:6642\n35d2b7a5-e3a7-485a-a4b7-b4970eb0e63b    cluster-tcp-sess    tcp        10.100.158.128:8080     10.16.0.10:8080,10.16.0.5:8080,10.16.63.30:8080\n                                                            tcp        10.107.26.215:8080      10.16.0.19:8080,10.16.0.20:8080,10.16.0.21:8080\n                                                            tcp        10.107.26.215:9093      10.16.0.19:9093,10.16.0.20:9093,10.16.0.21:9093\n                                                            tcp        10.98.187.99:8080       10.16.0.22:8080,10.16.0.23:8080\n                                                            tcp        10.98.187.99:9090       10.16.0.22:9090,10.16.0.23:9090\nf43303e4-89aa-4d3e-a3dc-278a552fe27b    cluster-udp-load    udp        10.96.0.10:53           10.16.0.4:53,10.16.0.9:53\n_uuid               : 06776304-5a96-43ed-90c4-c4854c251699\naddresses           : []\nexternal_ids        : {vendor=kube-ovn}\nname                : node_liumengxin_ovn2_192.168.137.177_underlay_v6\n\n_uuid               : 62690625-87d5-491c-8675-9fd83b1f433c\naddresses           : []\nexternal_ids        : {vendor=kube-ovn}\nname                : node_liumengxin_ovn1_192.168.137.176_underlay_v6\n\n_uuid               : b03a9bae-94d5-4562-b34c-b5f6198e180b\naddresses           : [\"10.16.0.0/16\", \"100.64.0.0/16\", \"172.22.0.0/16\", \"192.101.0.0/16\"]\nexternal_ids        : {vendor=kube-ovn}\nname                : ovn.cluster.overlay.subnets.IPv4\n\n_uuid               : e1056f3a-24cc-4666-8a91-75ee6c3c2426\naddresses           : []\nexternal_ids        : {vendor=kube-ovn}\nname                : ovn.cluster.overlay.subnets.IPv6\n\n_uuid               : 3e5d5fff-e670-47b2-a2f5-a39f4698a8c5\naddresses           : []\nexternal_ids        : {vendor=kube-ovn}\nname                : node_liumengxin_ovn3_192.168.137.178_underlay_v6\n_uuid               : 2d85dbdc-d0db-4abe-b19e-cc806d32b492\naction              : drop\ndirection           : from-lport\nexternal_ids        : {}\nlabel               : 0\nlog                 : false\nmatch               : \"inport==@ovn.sg.kubeovn_deny_all &amp;&amp; ip\"\nmeter               : []\nname                : []\noptions             : {}\npriority            : 2003\nseverity            : []\n\n_uuid               : de790cc8-f155-405f-bb32-5a51f30c545f\naction              : drop\ndirection           : to-lport\nexternal_ids        : {}\nlabel               : 0\nlog                 : false\nmatch               : \"outport==@ovn.sg.kubeovn_deny_all &amp;&amp; ip\"\nmeter               : []\nname                : []\noptions             : {}\npriority            : 2003\nseverity            : []\nChassis \"e15ed4d4-1780-4d50-b09e-ea8372ed48b8\"\nhostname: liumengxin-ovn1-192.168.137.176\n    Encap stt\n        ip: \"192.168.137.176\"\noptions: {csum=\"true\"}\nPort_Binding node-liumengxin-ovn1-192.168.137.176\n    Port_Binding perf-6vxkn.default\n    Port_Binding kube-state-metrics-5d6885d89-4nf8h.monitoring\n    Port_Binding alertmanager-main-0.monitoring\n    Port_Binding kube-ovn-pinger-6ftdf.kube-system\n    Port_Binding fake-kubelet-67c55dfd89-pv86k.kube-system\n    Port_Binding prometheus-k8s-0.monitoring\nChassis \"eef07da1-f8ad-4775-b14d-bd6a3b4eb0d5\"\nhostname: liumengxin-ovn3-192.168.137.178\n    Encap stt\n        ip: \"192.168.137.178\"\noptions: {csum=\"true\"}\nPort_Binding kube-ovn-pinger-7twb4.kube-system\n    Port_Binding prometheus-adapter-86df476d87-rl88g.monitoring\n    Port_Binding prometheus-k8s-1.monitoring\n    Port_Binding node-liumengxin-ovn3-192.168.137.178\n    Port_Binding perf-ff475.default\n    Port_Binding alertmanager-main-1.monitoring\n    Port_Binding blackbox-exporter-676d976865-tvsjd.monitoring\nChassis \"efa253c9-494d-4719-83ae-b48ab0f11c03\"\nhostname: liumengxin-ovn2-192.168.137.177\n    Encap stt\n        ip: \"192.168.137.177\"\noptions: {csum=\"true\"}\nPort_Binding grafana-6c4c6b8fb7-pzd2c.monitoring\n    Port_Binding node-liumengxin-ovn2-192.168.137.177\n    Port_Binding alertmanager-main-2.monitoring\n    Port_Binding coredns-6789c94dd8-9jqsz.kube-system\n    Port_Binding coredns-6789c94dd8-25d4r.kube-system\n    Port_Binding prometheus-operator-7bbc99fc8b-wgjm4.monitoring\n    Port_Binding prometheus-adapter-86df476d87-gdxmc.monitoring\n    Port_Binding perf-fjnws.default\n    Port_Binding kube-ovn-pinger-vh2xg.kube-system\nds kube-proxy ready\nkube-proxy ready\ndeployment ovn-central ready\ndeployment kube-ovn-controller ready\nds kube-ovn-cni ready\nds ovs-ovn ready\ndeployment coredns ready\novn-nb leader check ok\novn-sb leader check ok\novn-northd leader check ok\n### kube-ovn-controller recent log\n\n### start to diagnose node liumengxin-ovn1-192.168.137.176\n#### ovn-controller log:\n2022-06-03T00:56:44.897Z|16722|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T01:06:44.912Z|16723|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T01:16:44.925Z|16724|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T01:26:44.936Z|16725|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T01:36:44.959Z|16726|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T01:46:44.974Z|16727|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T01:56:44.988Z|16728|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T02:06:45.001Z|16729|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T02:16:45.025Z|16730|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T02:26:45.040Z|16731|inc_proc_eng|INFO|User triggered force recompute.\n\n#### ovs-vswitchd log:\n2022-06-02T23:03:00.137Z|00079|dpif(handler1)|WARN|system@ovs-system: execute ct(commit,zone=14,label=0/0x1,nat(src)),8 failed (Invalid argument) on packet icmp,vlan_tci=0x0000,dl_src=00:00:00:f8:07:c8,dl_dst=00:00:00:fa:1e:50,nw_src=10.16.0.5,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0 icmp_csum:f9d1\n with metadata skb_priority(0),tunnel(tun_id=0x160017000004,src=192.168.137.177,dst=192.168.137.176,ttl=64,tp_src=38881,tp_dst=7471,flags(csum|key)),skb_mark(0),ct_state(0x21),ct_zone(0xe),ct_tuple4(src=10.16.0.5,dst=10.16.0.10,proto=1,tp_src=8,tp_dst=0),in_port(4) mtu 0\n2022-06-02T23:23:31.840Z|00080|dpif(handler1)|WARN|system@ovs-system: execute ct(commit,zone=14,label=0/0x1,nat(src)),8 failed (Invalid argument) on packet icmp,vlan_tci=0x0000,dl_src=00:00:00:f8:07:c8,dl_dst=00:00:00:fa:1e:50,nw_src=10.16.0.5,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0 icmp_csum:15b2\n with metadata skb_priority(0),tunnel(tun_id=0x160017000004,src=192.168.137.177,dst=192.168.137.176,ttl=64,tp_src=38881,tp_dst=7471,flags(csum|key)),skb_mark(0),ct_state(0x21),ct_zone(0xe),ct_tuple4(src=10.16.0.5,dst=10.16.0.10,proto=1,tp_src=8,tp_dst=0),in_port(4) mtu 0\n2022-06-03T00:09:15.659Z|00081|dpif(handler1)|WARN|system@ovs-system: execute ct(commit,zone=14,label=0/0x1,nat(src)),8 failed (Invalid argument) on packet icmp,vlan_tci=0x0000,dl_src=00:00:00:dc:e3:63,dl_dst=00:00:00:fa:1e:50,nw_src=10.16.63.30,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0 icmp_csum:e5a5\n with metadata skb_priority(0),tunnel(tun_id=0x150017000004,src=192.168.137.178,dst=192.168.137.176,ttl=64,tp_src=9239,tp_dst=7471,flags(csum|key)),skb_mark(0),ct_state(0x21),ct_zone(0xe),ct_tuple4(src=10.16.63.30,dst=10.16.0.10,proto=1,tp_src=8,tp_dst=0),in_port(4) mtu 0\n2022-06-03T00:30:13.409Z|00064|dpif(handler2)|WARN|system@ovs-system: execute ct(commit,zone=14,label=0/0x1,nat(src)),8 failed (Invalid argument) on packet icmp,vlan_tci=0x0000,dl_src=00:00:00:f8:07:c8,dl_dst=00:00:00:fa:1e:50,nw_src=10.16.0.5,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0 icmp_csum:6b4a\n with metadata skb_priority(0),tunnel(tun_id=0x160017000004,src=192.168.137.177,dst=192.168.137.176,ttl=64,tp_src=38881,tp_dst=7471,flags(csum|key)),skb_mark(0),ct_state(0x21),ct_zone(0xe),ct_tuple4(src=10.16.0.5,dst=10.16.0.10,proto=1,tp_src=8,tp_dst=0),in_port(4) mtu 0\n2022-06-03T02:02:33.832Z|00082|dpif(handler1)|WARN|system@ovs-system: execute ct(commit,zone=14,label=0/0x1,nat(src)),8 failed (Invalid argument) on packet icmp,vlan_tci=0x0000,dl_src=00:00:00:f8:07:c8,dl_dst=00:00:00:fa:1e:50,nw_src=10.16.0.5,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0 icmp_csum:a819\n with metadata skb_priority(0),tunnel(tun_id=0x160017000004,src=192.168.137.177,dst=192.168.137.176,ttl=64,tp_src=38881,tp_dst=7471,flags(csum|key)),skb_mark(0),ct_state(0x21),ct_zone(0xe),ct_tuple4(src=10.16.0.5,dst=10.16.0.10,proto=1,tp_src=8,tp_dst=0),in_port(4) mtu 0\n\n#### ovs-vsctl show results:\n0d4c4675-c9cc-440a-8c1a-878e17f81b88\n    Bridge br-int\n        fail_mode: secure\n        datapath_type: system\n        Port a2c1a8a8b83a_h\n            Interface a2c1a8a8b83a_h\n        Port \"4fa5c4cbb1a5_h\"\nInterface \"4fa5c4cbb1a5_h\"\nPort ovn-eef07d-0\n            Interface ovn-eef07d-0\n                type: stt\n                options: {csum=\"true\", key=flow, remote_ip=\"192.168.137.178\"}\nPort ovn0\n            Interface ovn0\n                type: internal\n        Port \"04d03360e9a0_h\"\nInterface \"04d03360e9a0_h\"\nPort eeb4d9e51b5d_h\n            Interface eeb4d9e51b5d_h\n        Port mirror0\n            Interface mirror0\n                type: internal\n        Port \"8e5d887ccd80_h\"\nInterface \"8e5d887ccd80_h\"\nPort ovn-efa253-0\n            Interface ovn-efa253-0\n                type: stt\n                options: {csum=\"true\", key=flow, remote_ip=\"192.168.137.177\"}\nPort \"17512d5be1f1_h\"\nInterface \"17512d5be1f1_h\"\nPort br-int\n            Interface br-int\n                type: internal\n    ovs_version: \"2.17.2\"\n\n#### pinger diagnose results:\nI0603 10:35:04.349404   17619 pinger.go:19]\n-------------------------------------------------------------------------------\nKube-OVN:\n  Version:       v1.13.0\n  Build:         2022-04-24_08:02:50\n  Commit:        git-73f9d15\n  Go Version:    go1.17.8\n  Arch:          amd64\n-------------------------------------------------------------------------------\nI0603 10:35:04.376797   17619 config.go:166] pinger config is &amp;{KubeConfigFile: KubeClient:0xc000493380 Port:8080 DaemonSetNamespace:kube-system DaemonSetName:kube-ovn-pinger Interval:5 Mode:job ExitCode:0 InternalDNS:kubernetes.default ExternalDNS: NodeName:liumengxin-ovn1-192.168.137.176 HostIP:192.168.137.176 PodName:kube-ovn-pinger-6ftdf PodIP:10.16.0.10 PodProtocols:[IPv4] ExternalAddress: NetworkMode:kube-ovn PollTimeout:2 PollInterval:15 SystemRunDir:/var/run/openvswitch DatabaseVswitchName:Open_vSwitch DatabaseVswitchSocketRemote:unix:/var/run/openvswitch/db.sock DatabaseVswitchFileDataPath:/etc/openvswitch/conf.db DatabaseVswitchFileLogPath:/var/log/openvswitch/ovsdb-server.log DatabaseVswitchFilePidPath:/var/run/openvswitch/ovsdb-server.pid DatabaseVswitchFileSystemIDPath:/etc/openvswitch/system-id.conf ServiceVswitchdFileLogPath:/var/log/openvswitch/ovs-vswitchd.log ServiceVswitchdFilePidPath:/var/run/openvswitch/ovs-vswitchd.pid ServiceOvnControllerFileLogPath:/var/log/ovn/ovn-controller.log ServiceOvnControllerFilePidPath:/var/run/ovn/ovn-controller.pid}\nI0603 10:35:04.449166   17619 exporter.go:75] liumengxin-ovn1-192.168.137.176: exporter connect successfully\nI0603 10:35:04.554011   17619 ovn.go:21] ovs-vswitchd and ovsdb are up\nI0603 10:35:04.651293   17619 ovn.go:33] ovn_controller is up\nI0603 10:35:04.651342   17619 ovn.go:39] start to check port binding\nI0603 10:35:04.749613   17619 ovn.go:135] chassis id is 1d7f3d6c-eec5-4b3c-adca-2969d9cdfd80\nI0603 10:35:04.763487   17619 ovn.go:49] port in sb is [node-liumengxin-ovn1-192.168.137.176 perf-6vxkn.default kube-state-metrics-5d6885d89-4nf8h.monitoring alertmanager-main-0.monitoring kube-ovn-pinger-6ftdf.kube-system fake-kubelet-67c55dfd89-pv86k.kube-system prometheus-k8s-0.monitoring]\nI0603 10:35:04.763583   17619 ovn.go:61] ovs and ovn-sb binding check passed\nI0603 10:35:05.049309   17619 ping.go:259] start to check apiserver connectivity\nI0603 10:35:05.053666   17619 ping.go:268] connect to apiserver success in 4.27ms\nI0603 10:35:05.053786   17619 ping.go:129] start to check pod connectivity\nI0603 10:35:05.249590   17619 ping.go:159] ping pod: kube-ovn-pinger-6ftdf 10.16.0.10, count: 3, loss count 0, average rtt 16.30ms\nI0603 10:35:05.354135   17619 ping.go:159] ping pod: kube-ovn-pinger-7twb4 10.16.63.30, count: 3, loss count 0, average rtt 1.81ms\nI0603 10:35:05.458460   17619 ping.go:159] ping pod: kube-ovn-pinger-vh2xg 10.16.0.5, count: 3, loss count 0, average rtt 1.92ms\nI0603 10:35:05.458523   17619 ping.go:83] start to check node connectivity\n</code></pre> <p>\u5982\u679c diagnose \u7684\u76ee\u6807\u6307\u5b9a\u4e3a subnet \u8be5\u811a\u672c\u4f1a\u5728 subnet \u4e0a\u5efa\u7acb daemonset\uff0c\u7531 <code>kube-ovn-pinger</code> \u53bb\u63a2\u6d4b\u8fd9\u4e2a daemonset \u7684\u6240\u6709 pod \u7684\u8fde\u901a\u6027\u548c\u7f51\u7edc\u5ef6\u65f6\uff0c\u6d4b\u8bd5\u5b8c\u540e\u81ea\u52a8\u9500\u6bc1\u8be5 daemonset\u3002</p> <p>\u5982\u679c diagnose \u7684\u76ee\u6807\u6307\u5b9a\u4e3a IPPorts \u8be5\u811a\u672c\u4f1a\u8ba9\u6bcf\u4e2a <code>kube-ovn-pinger</code> pod \u53bb\u63a2\u6d4b\u76ee\u6807\u534f\u8bae\uff0cIP\uff0cPort \u662f\u5426\u53ef\u8fbe\u3002</p>"},{"location":"ops/kubectl-ko/#tuning-install-fastpathlocal-install-fastpathremove-fastpathinstall-sttlocal-install-sttremove-stt-centos7centos8-kernel-devel-version","title":"tuning {install-fastpath|local-install-fastpath|remove-fastpath|install-stt|local-install-stt|remove-stt} {centos7|centos8}} [kernel-devel-version]","text":"<p>\u8be5\u547d\u4ee4\u6267\u884c\u6027\u80fd\u8c03\u4f18\u76f8\u5173\u64cd\u4f5c\uff0c\u5177\u4f53\u4f7f\u7528\u8bf7\u53c2\u8003\u6027\u80fd\u8c03\u4f18\u3002</p>"},{"location":"ops/kubectl-ko/#reload","title":"reload","text":"<p>\u8be5\u547d\u4ee4\u91cd\u542f\u6240\u6709 Kube-OVN \u76f8\u5173\u7ec4\u4ef6\uff1a</p> <pre><code># kubectl ko reload\npod \"ovn-central-8684dd94bd-vzgcr\" deleted\nWaiting for deployment \"ovn-central\" rollout to finish: 0 of 1 updated replicas are available...\ndeployment \"ovn-central\" successfully rolled out\npod \"ovs-ovn-bsnvz\" deleted\npod \"ovs-ovn-m9b98\" deleted\npod \"kube-ovn-controller-8459db5ff4-64c62\" deleted\nWaiting for deployment \"kube-ovn-controller\" rollout to finish: 0 of 1 updated replicas are available...\ndeployment \"kube-ovn-controller\" successfully rolled out\npod \"kube-ovn-cni-2klnh\" deleted\npod \"kube-ovn-cni-t2jz4\" deleted\nWaiting for daemon set \"kube-ovn-cni\" rollout to finish: 0 of 2 updated pods are available...\nWaiting for daemon set \"kube-ovn-cni\" rollout to finish: 1 of 2 updated pods are available...\ndaemon set \"kube-ovn-cni\" successfully rolled out\npod \"kube-ovn-pinger-ln72z\" deleted\npod \"kube-ovn-pinger-w8lrk\" deleted\nWaiting for daemon set \"kube-ovn-pinger\" rollout to finish: 0 of 2 updated pods are available...\nWaiting for daemon set \"kube-ovn-pinger\" rollout to finish: 1 of 2 updated pods are available...\ndaemon set \"kube-ovn-pinger\" successfully rolled out\npod \"kube-ovn-monitor-7fb67d5488-7q6zb\" deleted\nWaiting for deployment \"kube-ovn-monitor\" rollout to finish: 0 of 1 updated replicas are available...\ndeployment \"kube-ovn-monitor\" successfully rolled out\n</code></pre>"},{"location":"ops/kubectl-ko/#log","title":"log","text":"<p>\u4f7f\u7528\u8be5\u547d\u4ee4\u4f1a\u6293\u53d6 kube-ovn \u6240\u6709\u8282\u70b9\u4e0a\u7684 Kube-OVN\uff0cOVN\uff0cOpenvswitch \u7684 log \u4ee5\u53ca linux \u5e38\u7528\u7684\u4e00\u4e9b debug \u4fe1\u606f\u3002</p> <pre><code># kubectl ko log all\nCollecting kube-ovn logging files\nCollecting ovn logging files\nCollecting openvswitch logging files\nCollecting linux dmesg files\nCollecting linux iptables-legacy files\nCollecting linux iptables-nft files\nCollecting linux route files\nCollecting linux link files\nCollecting linux neigh files\nCollecting linux memory files\nCollecting linux top files\nCollecting linux sysctl files\nCollecting linux netstat files\nCollecting linux addr files\nCollecting linux ipset files\nCollecting linux tcp files\nCollected files have been saved in the directory /root/kubectl-ko-log\n</code></pre> <p>\u76ee\u5f55\u5982\u4e0b\uff1a</p> <pre><code># tree kubectl-ko-log/\nkubectl-ko-log/\n|-- kube-ovn-control-plane\n|   |-- kube-ovn\n|   |   |-- kube-ovn-cni.log\n|   |   |-- kube-ovn-monitor.log\n|   |   `-- kube-ovn-pinger.log\n|   |-- linux\n|   |   |-- addr.log\n|   |   |-- dmesg.log\n|   |   |-- ipset.log\n|   |   |-- iptables-legacy.log\n|   |   |-- iptables-nft.log\n|   |   |-- link.log\n|   |   |-- memory.log\n|   |   |-- neigh.log\n|   |   |-- netstat.log\n|   |   |-- route.log\n|   |   |-- sysctl.log\n|   |   |-- tcp.log\n|   |   `-- top.log\n|   |-- openvswitch\n|   |   |-- ovs-vswitchd.log\n|   |   `-- ovsdb-server.log\n|   `-- ovn\n|       |-- ovn-controller.log\n|       |-- ovn-northd.log\n|       |-- ovsdb-server-nb.log\n|       `-- ovsdb-server-sb.log\n</code></pre>"},{"location":"ops/kubectl-ko/#perf-image","title":"perf [image]","text":"<p>\u8be5\u547d\u4ee4\u4f1a\u53bb\u6d4b\u8bd5 Kube-OVN \u7684\u4e00\u4e9b\u6027\u80fd\u6307\u6807\u5982\u4e0b\uff1a</p> <ol> <li>\u5bb9\u5668\u7f51\u7edc\u7684\u6027\u80fd\u6307\u6807\uff1b</li> <li>Hostnetwork \u7f51\u7edc\u6027\u80fd\u6307\u6807\uff1b</li> <li>\u5bb9\u5668\u7f51\u7edc\u7ec4\u64ad\u62a5\u6587\u6027\u80fd\u6307\u6807\uff1b</li> <li>OVN-NB, OVN-SB, OVN-Northd leader \u5220\u9664\u6062\u590d\u6240\u9700\u65f6\u95f4\u3002</li> </ol> <p>\u53c2\u6570 image \u7528\u4e8e\u6307\u5b9a\u6027\u80fd\u6d4b\u8bd5 pod \u6240\u7528\u7684\u955c\u50cf\uff0c\u9ed8\u8ba4\u60c5\u51b5\u4e0b\u662f <code>kubeovn/test:v1.12.0</code>, \u8bbe\u7f6e\u8be5\u53c2\u6570\u4e3b\u8981\u662f\u4e3a\u4e86\u79bb\u7ebf\u573a\u666f\uff0c\u5c06\u955c\u50cf\u62c9\u5230\u5185\u7f51\u73af\u5883\u53ef\u80fd\u4f1a\u6709\u955c\u50cf\u540d\u53d8\u5316\u3002</p> <pre><code># kubectl ko perf\n============================== Prepareing Performance Test Resources ===============================\npod/test-client created\npod/test-host-client created\npod/test-server created\npod/test-host-server created\nservice/test-server created\npod/test-client condition met\npod/test-host-client condition met\npod/test-host-server condition met\npod/test-server condition met\n====================================================================================================\n============================ Start Pod Network Unicast Performance Test ============================\nSize            TCP Latency     TCP Bandwidth   UDP Latency     UDP Lost Rate   UDP Bandwidth\n64              82.8 us         97.7 Mbits/sec  67.6 us         (0%)            8.42 Mbits/sec\n128             85.4 us         167 Mbits/sec   67.2 us         (0%)            17.2 Mbits/sec\n512             85.8 us         440 Mbits/sec   68.7 us         (0%)            68.4 Mbits/sec\n1k              85.1 us         567 Mbits/sec   68.7 us         (0%)            134 Mbits/sec\n4k              138 us          826 Mbits/sec   78.1 us         (1.4%)          503 Mbits/sec\n====================================================================================================\n=============================== Start Host Network Performance Test ================================\nSize            TCP Latency     TCP Bandwidth   UDP Latency     UDP Lost Rate   UDP Bandwidth\n64              49.7 us         120 Mbits/sec   37.9 us         (0%)            18.6 Mbits/sec\n128             49.7 us         200 Mbits/sec   38.1 us         (0%)            35.5 Mbits/sec\n512             51.9 us         588 Mbits/sec   38.9 us         (0%)            142 Mbits/sec\n1k              51.7 us         944 Mbits/sec   37.2 us         (0%)            279 Mbits/sec\n4k              74.9 us         1.66 Gbits/sec  39.9 us         (0%)            1.20 Gbits/sec\n====================================================================================================\n============================== Start Service Network Performance Test ==============================\nSize            TCP Latency     TCP Bandwidth   UDP Latency     UDP Lost Rate   UDP Bandwidth\n64              111 us          96.3 Mbits/sec  88.4 us         (0%)            7.59 Mbits/sec\n128             83.7 us         150 Mbits/sec   69.2 us         (0%)            16.9 Mbits/sec\n512             87.4 us         374 Mbits/sec   75.8 us         (0%)            60.9 Mbits/sec\n1k              88.2 us         521 Mbits/sec   73.1 us         (0%)            123 Mbits/sec\n4k              148 us          813 Mbits/sec   77.6 us         (0.0044%)       451 Mbits/sec\n====================================================================================================\n=========================== Start Pod Multicast Network Performance Test ===========================\nSize            UDP Latency     UDP Lost Rate   UDP Bandwidth\n64              0.014 ms        (0.17%)         5.80 Mbits/sec\n128             0.012 ms        (0%)            11.4 Mbits/sec\n512             0.016 ms        (0%)            46.1 Mbits/sec\n1k              0.023 ms        (0.073%)        89.8 Mbits/sec\n4k              0.035 ms        (1.3%)          126 Mbits/sec\n====================================================================================================\n============================= Start Host Multicast Network Performance =============================\nSize            UDP Latency     UDP Lost Rate   UDP Bandwidth\n64              0.007 ms        (0%)            9.95 Mbits/sec\n128             0.005 ms        (0%)            21.8 Mbits/sec\n512             0.008 ms        (0%)            86.8 Mbits/sec\n1k              0.013 ms        (0.045%)        168 Mbits/sec\n4k              0.010 ms        (0.31%)         242 Mbits/sec\n====================================================================================================\n================================== Start Leader Recover Time Test ==================================\nDelete ovn central nb pod\npod \"ovn-central-5cb9c67d75-tlz9w\" deleted\nWaiting for ovn central nb pod running\n=============================== OVN nb Recovery takes 3.305236803 s ================================\nDelete ovn central sb pod\npod \"ovn-central-5cb9c67d75-szx4c\" deleted\nWaiting for ovn central sb pod running\n=============================== OVN sb Recovery takes 3.462698535 s ================================\nDelete ovn central northd pod\npod \"ovn-central-5cb9c67d75-zqmqv\" deleted\nWaiting for ovn central northd pod running\n============================= OVN northd Recovery takes 2.691291403 s ==============================\n====================================================================================================\n================================= Remove Performance Test Resource =================================\nrm -f unicast-test-client.log\nrm -f unicast-test-host-client.log\nrm -f unicast-test-client.log\nkubectl ko nbctl lb-del test-server\nrm -f multicast-test-server.log\nkubectl exec ovs-ovn-gxdrf -n kube-system -- ip maddr del 01:00:5e:00:00:64 dev eth0\nkubectl exec ovs-ovn-h57bf -n kube-system -- ip maddr del 01:00:5e:00:00:64 dev eth0\nrm -f multicast-test-host-server.log\npod \"test-client\" deleted\npod \"test-host-client\" deleted\npod \"test-host-server\" deleted\npod \"test-server\" deleted\nservice \"test-server\" deleted\n====================================================================================================\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"ops/recover-db/","title":"OVN \u6570\u636e\u5e93\u5907\u4efd\u548c\u6062\u590d","text":"<p>\u672c\u6587\u6863\u4ecb\u7ecd\u5982\u4f55\u8fdb\u884c\u6570\u636e\u5e93\u5907\u4efd\uff0c\u4ee5\u53ca\u5728\u4e0d\u540c\u60c5\u51b5\u4e0b\u5982\u4f55\u901a\u8fc7\u5df2\u6709\u7684\u6570\u636e\u5e93\u6587\u4ef6\u8fdb\u884c\u96c6\u7fa4\u6062\u590d\u3002</p>"},{"location":"ops/recover-db/#_1","title":"\u6570\u636e\u5e93\u5907\u4efd","text":"<p>\u5229\u7528 kubectl \u63d2\u4ef6\u7684 backup \u547d\u4ee4\u53ef\u4ee5\u5bf9\u6570\u636e\u5e93\u6587\u4ef6\u8fdb\u884c\u5907\u4efd\uff0c\u4ee5\u7528\u4e8e\u6545\u969c\u65f6\u6062\u590d\uff1a</p> <pre><code># kubectl ko nb backup\ntar: Removing leading `/' from member names\nbackup ovn-nb db to /root/ovnnb_db.060223191654183154.backup\n\n# kubectl ko sb backup\ntar: Removing leading `/' from member names\nbackup ovn-nb db to /root/ovnsb_db.060223191654183154.backup\n</code></pre>"},{"location":"ops/recover-db/#_2","title":"\u96c6\u7fa4\u90e8\u5206\u6545\u969c\u6062\u590d","text":"<p>\u82e5\u96c6\u7fa4\u4e2d\u5b58\u5728\u90e8\u5206\u8282\u70b9\u56e0\u4e3a\u65ad\u7535\uff0c\u6587\u4ef6\u7cfb\u7edf\u6545\u969c\u6216\u78c1\u76d8\u7a7a\u95f4\u4e0d\u8db3\u5bfc\u81f4\u5de5\u4f5c\u5f02\u5e38\uff0c \u4f46\u662f\u96c6\u7fa4\u4ecd\u53ef\u6b63\u5e38\u5de5\u4f5c\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0b\u6b65\u9aa4\u8fdb\u884c\u6062\u590d\u3002</p>"},{"location":"ops/recover-db/#_3","title":"\u67e5\u770b\u65e5\u5fd7\u786e\u8ba4\u72b6\u6001\u5f02\u5e38","text":"<p>\u67e5\u770b\u5bf9\u5e94\u8282\u70b9 <code>/var/log/ovn/ovn-northd.log</code>\uff0c\u82e5\u63d0\u793a\u7c7b\u4f3c\u9519\u8bef\u5219\u53ef\u5224\u65ad\u6570\u636e\u5e93\u5b58\u5728\u5f02\u5e38</p> <pre><code> * ovn-northd is not running\novsdb-server: ovsdb error: error reading record 2739 from OVN_Northbound log: record 2739 advances commit index to 6308 but last log index is 6307\n* Starting ovsdb-nb\n</code></pre>"},{"location":"ops/recover-db/#_4","title":"\u4ece\u96c6\u7fa4\u4e2d\u8e22\u51fa\u5bf9\u5e94\u8282\u70b9","text":"<p>\u6839\u636e\u65e5\u5fd7\u63d0\u793a\u662f <code>OVN_Northbound</code> \u8fd8\u662f <code>OVN_Southbound</code> \u9009\u62e9\u5bf9\u5e94\u7684\u6570\u636e\u5e93\u8fdb\u884c\u64cd\u4f5c\u3002 \u4e0a\u8ff0\u65e5\u5fd7\u63d0\u793a\u4e3a <code>OVN_Northbound</code> \u5219\u5bf9 ovn-nb \u8fdb\u884c\u64cd\u4f5c\uff1a</p> <pre><code># kubectl ko nb status\n9182\nName: OVN_Northbound\nCluster ID: e75f (e75fa340-49ed-45ab-990e-26cb865ebc85)\nServer ID: 9182 (9182e8dd-b5b0-4dd8-8518-598cc1e374f3)\nAddress: tcp:[10.0.128.61]:6643\nStatus: cluster member\nRole: leader\nTerm: 1454\nLeader: self\nVote: self\n\nLast Election started 1732603 ms ago, reason: timeout\nLast Election won: 1732587 ms ago\nElection timer: 1000\nLog: [7332, 12512]\nEntries not yet committed: 1\nEntries not yet applied: 1\nConnections: -&gt;f080 &lt;-f080 &lt;-e631 -&gt;e631\nDisconnections: 1\nServers:\n    f080 (f080 at tcp:[10.0.129.139]:6643) next_index=12512 match_index=12510 last msg 63 ms ago\n    9182 (9182 at tcp:[10.0.128.61]:6643) (self) next_index=10394 match_index=12510\ne631 (e631 at tcp:[10.0.131.173]:6643) next_index=12512 match_index=0\n</code></pre> <p>\u4ece\u96c6\u7fa4\u4e2d\u8e22\u51fa\u72b6\u6001\u5f02\u5e38\u8282\u70b9\uff1a</p> <pre><code>kubectl ko nb kick e631\n</code></pre> <p>\u767b\u5f55\u5f02\u5e38\u8282\u70b9\uff0c\u5220\u9664\u5bf9\u5e94\u7684\u6570\u636e\u5e93\u6587\u4ef6\uff1a</p> <pre><code>mv /etc/origin/ovn/ovnnb_db.db /tmp\n</code></pre> <p>\u5220\u9664\u5bf9\u5e94\u8282\u70b9\u7684 <code>ovn-central</code> Pod\uff0c\u7b49\u5f85\u96c6\u7fa4\u81ea\u52a8\u6062\u590d\uff1a</p> <pre><code>kubectl delete pod -n kube-system ovn-central-xxxx\n</code></pre>"},{"location":"ops/recover-db/#_5","title":"\u96c6\u7fa4\u4e0d\u80fd\u6b63\u5e38\u5de5\u4f5c\u4e0b\u7684\u6062\u590d","text":"<p>\u82e5\u96c6\u7fa4\u591a\u6570\u8282\u70b9\u53d7\u635f\u65e0\u6cd5\u9009\u4e3e\u51fa leader\uff0c\u8bf7\u53c2\u7167\u4e0b\u9762\u7684\u6b65\u9aa4\u8fdb\u884c\u6062\u590d\u3002</p>"},{"location":"ops/recover-db/#ovn-central","title":"\u505c\u6b62 ovn-central","text":"<p>\u8bb0\u5f55\u5f53\u524d <code>ovn-central</code> \u526f\u672c\u6570\u91cf\uff0c\u5e76\u505c\u6b62 <code>ovn-central</code> \u907f\u514d\u65b0\u7684\u6570\u636e\u5e93\u53d8\u66f4\u5f71\u54cd\u6062\u590d\uff1a</p> <pre><code>kubectl scale deployment -n kube-system ovn-central --replicas=0\n</code></pre>"},{"location":"ops/recover-db/#_6","title":"\u9009\u62e9\u5907\u4efd","text":"<p>\u7531\u4e8e\u591a\u6570\u8282\u70b9\u53d7\u635f\uff0c\u9700\u8981\u4ece\u67d0\u4e2a\u6570\u636e\u5e93\u6587\u4ef6\u8fdb\u884c\u6062\u590d\u91cd\u5efa\u96c6\u7fa4\u3002\u5982\u679c\u4e4b\u524d\u5907\u4efd\u8fc7\u6570\u636e\u5e93 \u53ef\u4f7f\u7528\u4e4b\u524d\u7684\u5907\u4efd\u6587\u4ef6\u8fdb\u884c\u6062\u590d\u3002\u5982\u679c\u6ca1\u6709\u8fdb\u884c\u8fc7\u5907\u4efd\u53ef\u4ee5\u4f7f\u7528\u4e0b\u9762\u7684\u6b65\u9aa4\u4ece\u5df2\u6709\u7684\u6570\u636e\u5e93\u6587\u4ef6 \u4e2d\u751f\u6210\u4e00\u4e2a\u5907\u4efd\u3002</p> <p>\u7531\u4e8e\u9ed8\u8ba4\u6587\u4ef6\u5939\u4e0b\u7684\u6570\u636e\u5e93\u6587\u4ef6\u4e3a\u96c6\u7fa4\u683c\u5f0f\u6570\u636e\u5e93\u6587\u4ef6\uff0c\u5305\u542b\u5f53\u524d\u96c6\u7fa4\u7684\u4fe1\u606f\uff0c\u65e0\u6cd5\u76f4\u63a5 \u7528\u8be5\u6587\u4ef6\u91cd\u5efa\u6570\u636e\u5e93\uff0c\u9700\u8981\u4f7f\u7528 <code>ovsdb-tool cluster-to-standalone</code> \u8fdb\u884c\u683c\u5f0f\u8f6c\u6362\u3002</p> <p>\u9009\u62e9 <code>ovn-central</code> \u73af\u5883\u53d8\u91cf <code>NODE_IPS</code> \u4e2d\u6392\u7b2c\u4e00\u7684\u8282\u70b9\u6062\u590d\u6570\u636e\u5e93\u6587\u4ef6\uff0c \u5982\u679c\u7b2c\u4e00\u4e2a\u8282\u70b9\u6570\u636e\u5e93\u6587\u4ef6\u5df2\u635f\u574f\uff0c\u4ece\u5176\u4ed6\u673a\u5668 <code>/etc/origin/ovn</code> \u4e0b\u590d\u5236\u6587\u4ef6\u5230\u7b2c\u4e00\u53f0\u673a\u5668 \uff0c \u6267\u884c\u4e0b\u5217\u547d\u4ee4\u751f\u6210\u6570\u636e\u5e93\u6587\u4ef6\u5907\u4efd\u3002</p> <pre><code>docker run -it -v /etc/origin/ovn:/etc/ovn kubeovn/kube-ovn:v1.13.0 bash\ncd /etc/ovn/\novsdb-tool cluster-to-standalone ovnnb_db_standalone.db ovnnb_db.db\novsdb-tool cluster-to-standalone ovnsb_db_standalone.db ovnsb_db.db\n</code></pre>"},{"location":"ops/recover-db/#ovn-central_1","title":"\u5220\u9664\u6bcf\u4e2a ovn-central \u8282\u70b9\u4e0a\u7684\u6570\u636e\u5e93\u6587\u4ef6","text":"<p>\u4e3a\u4e86\u907f\u514d\u91cd\u5efa\u96c6\u7fa4\u65f6\u4f7f\u7528\u5230\u9519\u8bef\u7684\u6570\u636e\uff0c\u9700\u8981\u5bf9\u5df2\u6709\u6570\u636e\u5e93\u6587\u4ef6\u8fdb\u884c\u6e05\u7406\uff1a</p> <pre><code>mv /etc/origin/ovn/ovnnb_db.db /tmp\nmv /etc/origin/ovn/ovnsb_db.db /tmp\n</code></pre>"},{"location":"ops/recover-db/#_7","title":"\u6062\u590d\u6570\u636e\u5e93\u96c6\u7fa4","text":"<p>\u5c06\u5907\u4efd\u6570\u636e\u5e93\u5206\u522b\u91cd\u547d\u540d\u4e3a <code>ovnnb_db.db</code> \u548c <code>ovnsb_db.db</code>\uff0c\u5e76\u590d\u5236\u5230 <code>ovn-central</code>  \u73af\u5883\u53d8\u91cf <code>NODE_IPS</code> \u4e2d\u6392\u7b2c\u4e00\u673a\u5668\u7684 <code>/etc/origin/ovn/</code> \u76ee\u5f55\u4e0b\uff1a</p> <pre><code>mv /etc/origin/ovn/ovnnb_db_standalone.db /etc/origin/ovn/ovnnb_db.db\nmv /etc/origin/ovn/ovnsb_db_standalone.db /etc/origin/ovn/ovnsb_db.db\n</code></pre> <p>\u6062\u590d <code>ovn-central</code> \u7684\u526f\u672c\u6570\uff1a</p> <pre><code>kubectl scale deployment -n kube-system ovn-central --replicas=3\nkubectl rollout status deployment/ovn-central -n kube-system\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"reference/architecture/","title":"\u603b\u4f53\u67b6\u6784","text":"<p>\u672c\u6587\u6863\u5c06\u4ecb\u7ecd Kube-OVN \u7684\u603b\u4f53\u67b6\u6784\uff0c\u548c\u5404\u4e2a\u7ec4\u4ef6\u7684\u529f\u80fd\u4ee5\u53ca\u5176\u4e4b\u95f4\u7684\u4ea4\u4e92\u3002</p> <p>\u603b\u4f53\u6765\u770b\uff0cKube-OVN \u4f5c\u4e3a Kubernetes \u548c OVN \u4e4b\u95f4\u7684\u4e00\u4e2a\u6865\u6881\uff0c\u5c06\u6210\u719f\u7684 SDN \u548c\u4e91\u539f\u751f\u76f8\u7ed3\u5408\u3002 \u8fd9\u610f\u5473\u7740 Kube-OVN \u4e0d\u4ec5\u901a\u8fc7 OVN \u5b9e\u73b0\u4e86 Kubernetes \u4e0b\u7684\u7f51\u7edc\u89c4\u8303\uff0c\u4f8b\u5982 CNI\uff0cService \u548c Networkpolicy\uff0c\u8fd8\u5c06\u5927\u91cf\u7684 SDN \u9886\u57df\u80fd\u529b\u5e26\u5165\u4e91\u539f\u751f\uff0c\u4f8b\u5982\u903b\u8f91\u4ea4\u6362\u673a\uff0c\u903b\u8f91\u8def\u7531\u5668\uff0cVPC\uff0c\u7f51\u5173\uff0cQoS\uff0cACL \u548c\u6d41\u91cf\u955c\u50cf\u3002</p> <p>\u540c\u65f6 Kube-OVN \u8fd8\u4fdd\u6301\u4e86\u826f\u597d\u7684\u5f00\u653e\u6027\u53ef\u4ee5\u548c\u8bf8\u591a\u6280\u672f\u65b9\u6848\u96c6\u6210\uff0c\u4f8b\u5982 Cilium\uff0cSubmariner\uff0cPrometheus\uff0cKubeVirt \u7b49\u7b49\u3002</p>"},{"location":"reference/architecture/#_2","title":"\u7ec4\u4ef6\u4ecb\u7ecd","text":"<p>Kube-OVN \u7684\u7ec4\u4ef6\u53ef\u4ee5\u5927\u81f4\u5206\u4e3a\u4e09\u7c7b\uff1a</p> <ul> <li>\u4e0a\u6e38 OVN/OVS \u7ec4\u4ef6\u3002</li> <li>\u6838\u5fc3\u63a7\u5236\u5668\u548c Agent\u3002</li> <li>\u76d1\u63a7\uff0c\u8fd0\u7ef4\u5de5\u5177\u548c\u6269\u5c55\u7ec4\u4ef6\u3002</li> </ul> <p></p>"},{"location":"reference/architecture/#ovnovs","title":"\u4e0a\u6e38 OVN/OVS \u7ec4\u4ef6","text":"<p>\u8be5\u7c7b\u578b\u7ec4\u4ef6\u6765\u81ea OVN/OVS \u793e\u533a\uff0c\u5e76\u9488\u5bf9 Kube-OVN \u7684\u4f7f\u7528\u573a\u666f\u505a\u4e86\u7279\u5b9a\u4fee\u6539\u3002 OVN/OVS \u672c\u8eab\u662f\u4e00\u5957\u6210\u719f\u7684\u7ba1\u7406\u865a\u673a\u548c\u5bb9\u5668\u7684 SDN \u7cfb\u7edf\uff0c\u6211\u4eec\u5f3a\u70c8\u5efa\u8bae \u5bf9 Kube-OVN \u5b9e\u73b0\u611f\u5174\u8da3\u7684\u7528\u6237\u5148\u53bb\u8bfb\u4e00\u4e0b ovn-architecture(7) \u6765\u4e86\u89e3\u4ec0\u4e48\u662f OVN \u4ee5\u53ca \u5982\u4f55\u548c\u5b83\u8fdb\u884c\u96c6\u6210\u3002Kube-OVN \u4f7f\u7528 OVN \u7684\u5317\u5411\u63a5\u53e3\u521b\u5efa\u548c\u8c03\u6574\u865a\u62df\u7f51\u7edc\uff0c\u5e76\u5c06\u5176\u4e2d\u7684\u7f51\u7edc\u6982\u5ff5\u6620\u5c04\u5230 Kubernetes \u4e4b\u5185\u3002</p> <p>\u6240\u6709 OVN/OVS \u76f8\u5173\u7ec4\u4ef6\u90fd\u5df2\u6253\u5305\u6210\u5bf9\u5e94\u955c\u50cf\uff0c\u5e76\u53ef\u5728 Kubernetes \u4e2d\u8fd0\u884c\u3002</p>"},{"location":"reference/architecture/#ovn-central","title":"ovn-central","text":"<p><code>ovn-central</code> Deployment \u8fd0\u884c OVN \u7684\u7ba1\u7406\u5e73\u9762\u7ec4\u4ef6\uff0c\u5305\u62ec <code>ovn-nb</code>, <code>ovn-sb</code>, \u548c <code>ovn-northd</code>\u3002</p> <ul> <li><code>ovn-nb</code>\uff1a \u4fdd\u5b58\u865a\u62df\u7f51\u7edc\u914d\u7f6e\uff0c\u5e76\u63d0\u4f9b API \u8fdb\u884c\u865a\u62df\u7f51\u7edc\u7ba1\u7406\u3002<code>kube-ovn-controller</code> \u5c06\u4f1a\u4e3b\u8981\u548c <code>ovn-nb</code> \u8fdb\u884c\u4ea4\u4e92\u914d\u7f6e\u865a\u62df\u7f51\u7edc\u3002</li> <li><code>ovn-sb</code>\uff1a \u4fdd\u5b58\u4ece <code>ovn-nb</code> \u7684\u903b\u8f91\u7f51\u7edc\u751f\u6210\u7684\u903b\u8f91\u6d41\u8868\uff0c\u4ee5\u53ca\u5404\u4e2a\u8282\u70b9\u7684\u5b9e\u9645\u7269\u7406\u7f51\u7edc\u72b6\u6001\u3002</li> <li><code>ovn-northd</code>\uff1a\u5c06 <code>ovn-nb</code> \u7684\u865a\u62df\u7f51\u7edc\u7ffb\u8bd1\u6210 <code>ovn-sb</code> \u4e2d\u7684\u903b\u8f91\u6d41\u8868\u3002</li> </ul> <p>\u591a\u4e2a <code>ovn-central</code> \u5b9e\u4f8b\u4f1a\u901a\u8fc7 Raft \u534f\u8bae\u540c\u6b65\u6570\u636e\u4fdd\u8bc1\u9ad8\u53ef\u7528\u3002</p>"},{"location":"reference/architecture/#ovs-ovn","title":"ovs-ovn","text":"<p><code>ovs-ovn</code> \u4ee5 DaemonSet \u5f62\u5f0f\u8fd0\u884c\u5728\u6bcf\u4e2a\u8282\u70b9\uff0c\u5728 Pod \u5185\u8fd0\u884c\u4e86 <code>openvswitch</code>, <code>ovsdb</code>, \u548c <code>ovn-controller</code>\u3002\u8fd9\u4e9b\u7ec4\u4ef6\u4f5c\u4e3a <code>ovn-central</code> \u7684 Agent \u5c06\u903b\u8f91\u6d41\u8868\u7ffb\u8bd1\u6210\u771f\u5b9e\u7684\u7f51\u7edc\u914d\u7f6e\u3002</p>"},{"location":"reference/architecture/#agent","title":"\u6838\u5fc3\u63a7\u5236\u5668\u548c Agent","text":"<p>\u8be5\u90e8\u5206\u4e3a Kube-OVN \u7684\u6838\u5fc3\u7ec4\u4ef6\uff0c\u4f5c\u4e3a OVN \u548c Kubernetes \u4e4b\u95f4\u7684\u4e00\u4e2a\u6865\u6881\uff0c\u5c06\u4e24\u4e2a\u7cfb\u7edf\u6253\u901a\u5e76\u5c06\u7f51\u7edc\u6982\u5ff5\u8fdb\u884c\u76f8\u4e92\u8f6c\u6362\u3002 \u5927\u90e8\u5206\u7684\u6838\u5fc3\u529f\u80fd\u90fd\u5728\u8be5\u90e8\u5206\u7ec4\u4ef6\u4e2d\u5b9e\u73b0\u3002</p>"},{"location":"reference/architecture/#kube-ovn-controller","title":"kube-ovn-controller","text":"<p>\u8be5\u7ec4\u4ef6\u4e3a\u4e00\u4e2a Deployment \u6267\u884c\u6240\u6709 Kubernetes \u5185\u8d44\u6e90\u5230 OVN \u8d44\u6e90\u7684\u7ffb\u8bd1\u5de5\u4f5c\uff0c\u5176\u4f5c\u7528\u76f8\u5f53\u4e8e\u6574\u4e2a Kube-OVN \u7cfb\u7edf\u7684\u63a7\u5236\u5e73\u9762\u3002 <code>kube-ovn-controller</code> \u76d1\u542c\u4e86\u6240\u6709\u548c\u7f51\u7edc\u529f\u80fd\u76f8\u5173\u8d44\u6e90\u7684\u4e8b\u4ef6\uff0c\u5e76\u6839\u636e\u8d44\u6e90\u53d8\u5316\u60c5\u51b5\u66f4\u65b0 OVN \u5185\u7684\u903b\u8f91\u7f51\u7edc\u3002\u4e3b\u8981\u76d1\u542c\u7684\u8d44\u6e90\u5305\u62ec\uff1a Pod\uff0cService\uff0cEndpoint\uff0cNode\uff0cNetworkPolicy\uff0cVPC\uff0cSubnet\uff0cVlan\uff0cProviderNetwork\u3002</p> <p>\u4ee5 Pod \u4e8b\u4ef6\u4e3a\u4f8b\uff0c <code>kube-ovn-controller</code> \u76d1\u542c\u5230 Pod \u521b\u5efa\u4e8b\u4ef6\u540e\uff0c\u901a\u8fc7\u5185\u7f6e\u7684\u5185\u5b58 IPAM \u529f\u80fd\u5206\u914d\u5730\u5740\uff0c\u5e76\u8c03\u7528 <code>ovn-central</code> \u521b\u5efa \u903b\u8f91\u7aef\u53e3\uff0c\u9759\u6001\u8def\u7531\u548c\u53ef\u80fd\u7684 ACL \u89c4\u5219\u3002\u63a5\u4e0b\u6765 <code>kube-ovn-controller</code> \u5c06\u5206\u914d\u5230\u7684\u5730\u5740\uff0c\u548c\u5b50\u7f51\u4fe1\u606f\u4f8b\u5982 CIDR\uff0c\u7f51\u5173\uff0c\u8def\u7531\u7b49\u4fe1\u606f\u5199\u4f1a\u5230 Pod \u7684 annotation \u4e2d\u3002\u8be5 annotation \u540e\u7eed\u4f1a\u88ab <code>kube-ovn-cni</code> \u8bfb\u53d6\u7528\u6765\u914d\u7f6e\u672c\u5730\u7f51\u7edc\u3002</p>"},{"location":"reference/architecture/#kube-ovn-cni","title":"kube-ovn-cni","text":"<p>\u8be5\u7ec4\u4ef6\u4e3a\u4e00\u4e2a DaemonSet \u8fd0\u884c\u5728\u6bcf\u4e2a\u8282\u70b9\u4e0a\uff0c\u5b9e\u73b0 CNI \u63a5\u53e3\uff0c\u5e76\u64cd\u4f5c\u672c\u5730\u7684 OVS \u914d\u7f6e\u5355\u673a\u7f51\u7edc\u3002</p> <p>\u8be5 DaemonSet \u4f1a\u590d\u5236 <code>kube-ovn</code> \u4e8c\u8fdb\u5236\u6587\u4ef6\u5230\u6bcf\u53f0\u673a\u5668\uff0c\u4f5c\u4e3a <code>kubelet</code> \u548c <code>kube-ovn-cni</code> \u4e4b\u95f4\u7684\u4ea4\u4e92\u5de5\u5177\uff0c\u5c06\u76f8\u5e94 CNI \u8bf7\u6c42 \u53d1\u9001\u7ed9 <code>kube-ovn-cni</code> \u6267\u884c\u3002\u8be5\u4e8c\u8fdb\u5236\u6587\u4ef6\u9ed8\u8ba4\u4f1a\u88ab\u590d\u5236\u5230 <code>/opt/cni/bin</code> \u76ee\u5f55\u4e0b\u3002</p> <p><code>kube-ovn-cni</code> \u4f1a\u914d\u7f6e\u5177\u4f53\u7684\u7f51\u7edc\u6765\u6267\u884c\u76f8\u5e94\u6d41\u91cf\u64cd\u4f5c\uff0c\u4e3b\u8981\u5de5\u4f5c\u5305\u62ec\uff1a</p> <ol> <li>\u914d\u7f6e <code>ovn-controller</code> \u548c <code>vswitchd</code>\u3002</li> <li>\u5904\u7406 CNI add/del \u8bf7\u6c42\uff1a<ol> <li>\u521b\u5efa\u5220\u9664 veth \u5e76\u548c OVS \u7aef\u53e3\u7ed1\u5b9a\u3002</li> <li>\u914d\u7f6e OVS \u7aef\u53e3\u4fe1\u606f\u3002</li> <li>\u66f4\u65b0\u5bbf\u4e3b\u673a\u7684 iptables/ipset/route \u7b49\u89c4\u5219\u3002</li> </ol> </li> <li>\u52a8\u6001\u66f4\u65b0\u5bb9\u5668 QoS.</li> <li>\u521b\u5efa\u5e76\u914d\u7f6e <code>ovn0</code> \u7f51\u5361\u8054\u901a\u5bb9\u5668\u7f51\u7edc\u548c\u4e3b\u673a\u7f51\u7edc\u3002</li> <li>\u914d\u7f6e\u4e3b\u673a\u7f51\u5361\u6765\u5b9e\u73b0 Vlan/Underlay/EIP \u7b49\u529f\u80fd\u3002</li> <li>\u52a8\u6001\u914d\u7f6e\u96c6\u7fa4\u4e92\u8054\u7f51\u5173\u3002</li> </ol>"},{"location":"reference/architecture/#_3","title":"\u76d1\u63a7\uff0c\u8fd0\u7ef4\u5de5\u5177\u548c\u6269\u5c55\u7ec4\u4ef6","text":"<p>\u8be5\u90e8\u5206\u7ec4\u4ef6\u4e3b\u8981\u63d0\u4f9b\u76d1\u63a7\uff0c\u8bca\u65ad\uff0c\u8fd0\u7ef4\u64cd\u4f5c\u4ee5\u53ca\u548c\u5916\u90e8\u8fdb\u884c\u5bf9\u63a5\uff0c\u5bf9 Kube-OVN \u7684\u6838\u5fc3\u7f51\u7edc\u80fd\u529b\u8fdb\u884c\u6269\u5c55\uff0c\u5e76\u7b80\u5316\u65e5\u5e38\u8fd0\u7ef4\u64cd\u4f5c\u3002</p>"},{"location":"reference/architecture/#kube-ovn-speaker","title":"kube-ovn-speaker","text":"<p>\u8be5\u7ec4\u4ef6\u4e3a\u4e00\u4e2a DaemonSet \u8fd0\u884c\u5728\u7279\u5b9a\u6807\u7b7e\u7684\u8282\u70b9\u4e0a\uff0c\u5bf9\u5916\u53d1\u5e03\u5bb9\u5668\u7f51\u7edc\u7684\u8def\u7531\uff0c\u4f7f\u5f97\u5916\u90e8\u53ef\u4ee5\u76f4\u63a5\u901a\u8fc7 Pod IP \u8bbf\u95ee\u5bb9\u5668\u3002</p> <p>\u66f4\u591a\u76f8\u5173\u4f7f\u7528\u65b9\u5f0f\u8bf7\u53c2\u8003 BGP \u652f\u6301\u3002</p>"},{"location":"reference/architecture/#kube-ovn-pinger","title":"kube-ovn-pinger","text":"<p>\u8be5\u7ec4\u4ef6\u4e3a\u4e00\u4e2a DaemonSet \u8fd0\u884c\u5728\u6bcf\u4e2a\u8282\u70b9\u4e0a\u6536\u96c6 OVS \u8fd0\u884c\u4fe1\u606f\uff0c\u8282\u70b9\u7f51\u7edc\u8d28\u91cf\uff0c\u7f51\u7edc\u5ef6\u8fdf\u7b49\u4fe1\u606f\uff0c\u6536\u96c6\u7684\u76d1\u63a7\u6307\u6807\u53ef\u53c2\u8003 Kube-OVN \u76d1\u63a7\u6307\u6807\u3002</p>"},{"location":"reference/architecture/#kube-ovn-monitor","title":"kube-ovn-monitor","text":"<p>\u8be5\u7ec4\u4ef6\u4e3a\u4e00\u4e2a Deployment \u6536\u96c6 OVN \u7684\u8fd0\u884c\u4fe1\u606f\uff0c\u6536\u96c6\u7684\u76d1\u63a7\u6307\u6807\u53ef\u53c2\u8003 Kube-OVN \u76d1\u63a7\u6307\u6807\u3002</p>"},{"location":"reference/architecture/#kubectl-ko","title":"kubectl-ko","text":"<p>\u8be5\u7ec4\u4ef6\u4e3a kubectl \u63d2\u4ef6\uff0c\u53ef\u4ee5\u5feb\u901f\u8fd0\u884c\u5e38\u89c1\u8fd0\u7ef4\u64cd\u4f5c\uff0c\u66f4\u591a\u4f7f\u7528\u8bf7\u53c2\u8003 kubectl \u63d2\u4ef6\u4f7f\u7528\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"reference/dev-env/","title":"\u5f00\u53d1\u73af\u5883\u6784\u5efa","text":""},{"location":"reference/dev-env/#_2","title":"\u73af\u5883\u51c6\u5907","text":"<p>Kube-OVN \u4f7f\u7528 Go 1.20 \u5f00\u53d1\u5e76\u4f7f\u7528 Go Modules \u7ba1\u7406\u4f9d\u8d56\uff0c \u8bf7\u786e\u8ba4\u73af\u5883\u53d8\u91cf <code>GO111MODULE=\"on\"</code>\u3002</p> <p>gosec \u88ab\u7528\u6765\u626b\u63cf\u4ee3\u7801\u5b89\u5168\u76f8\u5173\u95ee\u9898\uff0c\u9700\u8981\u5728\u5f00\u53d1\u73af\u5883\u5b89\u88c5\uff1a</p> <pre><code>go get github.com/securego/gosec/v2/cmd/gosec\n</code></pre> <p>\u4e3a\u4e86\u964d\u4f4e\u6700\u7ec8\u751f\u6210\u955c\u50cf\u5927\u5c0f\uff0cKube-OVN \u4f7f\u7528\u4e86\u90e8\u5206 Docker buildx \u8bd5\u9a8c\u7279\u6027\uff0c\u8bf7\u66f4\u65b0 Docker \u81f3\u6700\u65b0\u7248\u672c \u5e76\u5f00\u542f buildx:</p> <pre><code>docker buildx create --use\n</code></pre>"},{"location":"reference/dev-env/#_3","title":"\u6784\u5efa\u955c\u50cf","text":"<p>\u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u4e0b\u8f7d\u4ee3\u7801\uff0c\u5e76\u751f\u6210\u8fd0\u884c Kube-OVN \u6240\u9700\u955c\u50cf\uff1a</p> <pre><code>git clone https://github.com/kubeovn/kube-ovn.git\ncd kube-ovn\nmake release\n</code></pre> <p>\u5982\u9700\u6784\u5efa\u5728 ARM \u73af\u5883\u4e0b\u8fd0\u884c\u7684\u955c\u50cf\uff0c\u8bf7\u6267\u884c\u4e0b\u9762\u7684\u547d\u4ee4\uff1a</p> <pre><code>make release-arm\n</code></pre>"},{"location":"reference/dev-env/#base","title":"\u6784\u5efa base \u955c\u50cf","text":"<p>\u5982\u9700\u8981\u66f4\u6539\u64cd\u4f5c\u7cfb\u7edf\u7248\u672c\uff0c\u4f9d\u8d56\u5e93\uff0cOVS/OVN \u4ee3\u7801\u7b49\uff0c\u9700\u8981\u5bf9 base \u955c\u50cf\u8fdb\u884c\u91cd\u65b0\u6784\u5efa\u3002</p> <p>base \u955c\u50cf\u4f7f\u7528\u7684 Dockerfile \u4e3a <code>dist/images/Dockerfile.base</code>\u3002</p> <p>\u6784\u5efa\u65b9\u6cd5\uff1a</p> <pre><code># build x86 base image\nmake base-amd64\n\n# build arm base image\nmake base-arm64\n</code></pre>"},{"location":"reference/dev-env/#e2e","title":"\u8fd0\u884c E2E","text":"<p>Kube-OVN \u4f7f\u7528 KIND \u6784\u5efa\u672c\u5730 Kubernetes \u96c6\u7fa4\uff0cj2cli \u6e32\u67d3\u6a21\u677f\uff0c Ginkgo \u6765\u8fd0\u884c\u6d4b\u8bd5\u4ee3\u7801\u3002\u8bf7\u53c2\u8003\u76f8\u5173\u6587\u6863\u8fdb\u884c\u4f9d\u8d56\u5b89\u88c5\u3002</p> <p>\u672c\u5730\u6267\u884c E2E \u6d4b\u8bd5\uff1a</p> <pre><code>make kind-init\nmake kind-install\nmake e2e\n</code></pre> <p>\u5982\u9700\u8fd0\u884c Underlay E2E \u6d4b\u8bd5\uff0c\u6267\u884c\u4e0b\u5217\u547d\u4ee4\uff1a</p> <pre><code>make kind-init\nmake kind-install-underlay\nmake e2e-underlay-single-nic\n</code></pre> <p>\u5982\u9700\u8fd0\u884c ovn vpc nat gw eip, fip, snat, dnat \u6d4b\u8bd5\uff0c\u6267\u884c\u4e0b\u5217\u547d\u4ee4\uff1a</p> <pre><code>make kind-init\nmake kind-install\nmake ovn-vpc-nat-gw-conformance-e2e\n</code></pre> <p>\u5982\u9700\u8fd0\u884c iptables vpc nat gw eip, fip, snat, dnat \u6d4b\u8bd5\uff0c\u6267\u884c\u4e0b\u5217\u547d\u4ee4\uff1a</p> <pre><code>make kind-init\nmake kind-install\nmake kind-install-vpc-nat-gw\nmake iptables-vpc-nat-gw-conformance-e2e\n</code></pre> <p>\u5982\u9700\u8fd0\u884c loadbalancer service \u6d4b\u8bd5\uff0c\u6267\u884c\u4e0b\u5217\u547d\u4ee4\uff1a</p> <pre><code>make kind-init\nmake kind-install\nmake kind-install-lb-svc\nmake kube-ovn-lb-svc-conformance-e2e\n</code></pre> <p>\u5982\u9700\u6e05\u7406\uff0c\u6267\u884c\u4e0b\u5217\u547d\u4ee4\uff1a</p> <pre><code>make kind-clean\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"reference/document-convention/","title":"\u6587\u6863\u89c4\u8303","text":"<p>\u4e3a\u4e86\u4fdd\u8bc1\u6587\u6863\u98ce\u683c\u4e00\u81f4\uff0c\u8bf7\u5728\u63d0\u4ea4\u6587\u6863\u65f6\u9075\u5faa\u4e0b\u5217\u7684\u98ce\u683c\u89c4\u8303\u3002</p>"},{"location":"reference/document-convention/#_2","title":"\u6807\u70b9","text":"<p>\u4e2d\u6587\u6587\u6863\u4e2d\u6587\u672c\u5185\u5bb9\u6240\u6709\u6807\u70b9\u5e94\u4f7f\u7528\u4e2d\u6587\u683c\u5f0f\u6807\u70b9\uff0c\u82f1\u6587\u6587\u6863\u4e2d\u6240\u6709\u6587\u672c\u5185\u5bb9\u4e2d\u5e94\u4f7f\u7528\u82f1\u6587\u6807\u70b9\u3002</p> BadGood   \u8fd9\u91cc\u63d0\u4f9b\u4e86\u4e00\u952e\u5b89\u88c5\u811a\u672c,\u53ef\u4ee5\u5e2e\u52a9\u4f60\u5feb\u901f\u5b89\u88c5\u4e00\u4e2a\u9ad8\u53ef\u7528,\u751f\u4ea7\u5c31\u7eea\u7684\u5bb9\u5668\u7f51\u7edc.    \u8fd9\u91cc\u63d0\u4f9b\u4e86\u4e00\u952e\u5b89\u88c5\u811a\u672c\uff0c\u53ef\u4ee5\u5e2e\u52a9\u4f60\u5feb\u901f\u5b89\u88c5\u4e00\u4e2a\u9ad8\u53ef\u7528\uff0c\u751f\u4ea7\u5c31\u7eea\u7684\u5bb9\u5668\u7f51\u7edc\u3002   <p>\u82f1\u6587\u6570\u5b57\u548c\u4e2d\u6587\u5e94\u8be5\u7528\u7a7a\u683c\u8fdb\u884c\u5206\u9694\u3002</p> BadGood   Kube-OVN \u63d0\u4f9b\u4e86\u4e00\u952e\u5b89\u88c5\u811a\u672c\u6765\u5b89\u88c5 1.10 \u7248\u672c Kube-OVN\u3002    Kube-OVN \u63d0\u4f9b\u4e86\u4e00\u952e\u5b89\u88c5\u811a\u672c\u6765\u5b89\u88c5 1.10 \u7248\u672c Kube-OVN\u3002   <p>\u793a\u4f8b\u5185\u5bb9\u5e94\u8be5\u4ee5 <code>\uff1a</code> \u5f00\u542f\uff0c\u5176\u4ed6\u53e5\u5c3e\u9700\u8981\u7528 <code>\u3002</code> \u7ed3\u675f\u3002</p> BadGood   \u5b89\u88c5\u524d\u8bf7\u786e\u8ba4\u73af\u5883\u914d\u7f6e\u6b63\u786e  \u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u4e0b\u8f7d\u5b89\u88c5\u811a\u672c\u3002  <pre><code>wget 127.0.0.1\n</code></pre>   \u5b89\u88c5\u524d\u8bf7\u786e\u8ba4\u73af\u5883\u914d\u7f6e\u6b63\u786e\u3002  \u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u4e0b\u8f7d\u5b89\u88c5\u811a\u672c\uff1a  <pre><code>wget 127.0.0.1\n</code></pre>"},{"location":"reference/document-convention/#_3","title":"\u4ee3\u7801\u5757","text":"<p>yaml \u4ee3\u7801\u5757\u9700\u8981\u6807\u8bc6\u4e3a yaml\u3002</p> BadGood <pre><code>````\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n    name: attach-subnet\n````\n</code></pre> <pre><code>````yaml\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n    name: attach-subnet\n````\n</code></pre> <p>\u547d\u4ee4\u884c\u64cd\u4f5c\u793a\u4f8b\u4ee3\u7801\u5757\u9700\u8981\u6807\u8bc6\u4e3a bash\u3002</p> BadGood <pre><code>````\nwget 127.0.0.1\n````\n</code></pre> <pre><code>````bash\nwget 127.0.0.1\n````\n</code></pre> <p>\u5982\u679c\u547d\u4ee4\u884c\u64cd\u4f5c\u793a\u4f8b\u4e2d\u5305\u542b\u8f93\u51fa\u5185\u5bb9\uff0c\u5219\u6240\u6267\u884c\u547d\u4ee4\u9700\u8981\u4ee5 <code>#</code> \u5f00\u59cb\uff0c\u4ee5\u533a\u5206\u8f93\u5165\u4e0e\u8f93\u51fa\u3002</p> BadGood <pre><code>oilbeater@macdeMac-3 ~ ping 114.114.114.114 -c 3\nPING 114.114.114.114 (114.114.114.114): 56 data bytes\n64 bytes from 114.114.114.114: icmp_seq=0 ttl=83 time=10.429 ms\n64 bytes from 114.114.114.114: icmp_seq=1 ttl=79 time=11.360 ms\n64 bytes from 114.114.114.114: icmp_seq=2 ttl=76 time=10.794 ms\n\n--- 114.114.114.114 ping statistics ---\n3 packets transmitted, 3 packets received, 0.0% packet loss\nround-trip min/avg/max/stddev = 10.429/10.861/11.360/0.383 ms\n</code></pre> <pre><code># ping 114.114.114.114 -c 3\nPING 114.114.114.114 (114.114.114.114): 56 data bytes\n64 bytes from 114.114.114.114: icmp_seq=0 ttl=83 time=10.429 ms\n64 bytes from 114.114.114.114: icmp_seq=1 ttl=79 time=11.360 ms\n64 bytes from 114.114.114.114: icmp_seq=2 ttl=76 time=10.794 ms\n\n--- 114.114.114.114 ping statistics ---\n3 packets transmitted, 3 packets received, 0.0% packet loss\nround-trip min/avg/max/stddev = 10.429/10.861/11.360/0.383 ms\n</code></pre> <p>\u5982\u679c\u547d\u4ee4\u884c\u64cd\u4f5c\u793a\u4f8b\u4e2d\u53ea\u5305\u542b\u6267\u884c\u547d\u4ee4\uff0c\u6ca1\u6709\u8f93\u51fa\u7ed3\u679c\uff0c\u5219\u591a\u6761\u547d\u4ee4\u65e0\u9700 <code>#</code> \u5f00\u59cb\u3002</p> BadGood <pre><code># mv /etc/origin/ovn/ovnnb_db.db /tmp\n# mv /etc/origin/ovn/ovnsb_db.db /tmp\n</code></pre> <pre><code>mv /etc/origin/ovn/ovnnb_db.db /tmp\nmv /etc/origin/ovn/ovnsb_db.db /tmp\n</code></pre>"},{"location":"reference/document-convention/#_4","title":"\u94fe\u63a5","text":"<p>\u7ad9\u5185\u94fe\u63a5\u4f7f\u7528\u5bf9\u5e94 <code>md</code> \u6587\u4ef6\u8def\u5f84\u3002</p> BadGood <pre><code>\u5b89\u88c5\u524d\u8bf7\u53c2\u8003[\u51c6\u5907\u5de5\u4f5c](http://kubeovn.github.io/prepare)\u3002\n</code></pre> <pre><code>\u5b89\u88c5\u524d\u8bf7\u53c2\u8003[\u51c6\u5907\u5de5\u4f5c](./prepare.md)\u3002\n</code></pre> BadGood <pre><code>\u5982\u6709\u95ee\u9898\u8bf7\u53c2\u8003 [Kubernetes \u6587\u6863](http://kubernetes.io)\u3002\n</code></pre> <pre><code>\u5982\u6709\u95ee\u9898\u8bf7\u53c2\u8003 [Kubernetes \u6587\u6863](http://kubernetes.io){: target=\"_blank\" }\u3002\n</code></pre>"},{"location":"reference/document-convention/#_5","title":"\u7a7a\u884c","text":"<p>\u4e0d\u540c\u903b\u8f91\u5757\uff0c\u4f8b\u5982\u6807\u9898\u548c\u6587\u672c\uff0c\u6587\u672c\u548c\u4ee3\u7801\uff0c\u6587\u672c\u548c\u7f16\u53f7\u4e4b\u95f4\u9700\u8981\u7528\u7a7a\u884c\u5206\u9694\u3002</p> BadGood <pre><code>\u4e0b\u8f7d\u4e0b\u9762\u7684\u811a\u672c\uff0c\u8fdb\u884c\u5b89\u88c5\uff1a\n```bash\nwget 127.0.0.1\n```\n</code></pre> <pre><code>\u4e0b\u8f7d\u4e0b\u9762\u7684\u811a\u672c\uff0c\u8fdb\u884c\u5b89\u88c5\uff1a\n\n```bash\nwget 127.0.0.1\n```\n</code></pre> <p>\u4e0d\u540c\u903b\u8f91\u5757\u4e4b\u95f4\u53ea\u4f7f\u7528\u4e00\u4e2a\u7a7a\u884c\u8fdb\u884c\u5206\u9694\u3002</p> BadGood <pre><code>\u4e0b\u8f7d\u4e0b\u9762\u7684\u811a\u672c\uff0c\u8fdb\u884c\u5b89\u88c5\uff1a\n\n\n\n```bash\nwget 127.0.0.1\n```\n</code></pre> <pre><code>\u4e0b\u8f7d\u4e0b\u9762\u7684\u811a\u672c\uff0c\u8fdb\u884c\u5b89\u88c5\uff1a\n\n```bash\nwget 127.0.0.1\n```\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"reference/feature-stage/","title":"\u529f\u80fd\u6210\u719f\u5ea6","text":"<p>\u5728 Kube-OVN \u4e2d\u6839\u636e\u529f\u80fd\u4f7f\u7528\u5ea6\uff0c\u6587\u6863\u5b8c\u5584\u7a0b\u5ea6\u548c\u6d4b\u8bd5\u8986\u76d6\u7a0b\u5ea6\u5c06\u529f\u80fd\u6210\u719f\u5ea6\u5206\u4e3a Alpha\uff0cBeta \u548c GA \u4e09\u4e2a\u9636\u6bb5\u3002</p>"},{"location":"reference/feature-stage/#_2","title":"\u6210\u719f\u5ea6\u5b9a\u4e49","text":"<p>\u5bf9\u4e8e Alpha \u529f\u80fd\uff1a</p> <ul> <li>\u8be5\u529f\u80fd\u6ca1\u6709\u5b8c\u6574\u7684\u6587\u6863\u548c\u5b8c\u5584\u7684\u6d4b\u8bd5\u8986\u76d6\u3002</li> <li>\u8be5\u529f\u80fd\u672a\u6765\u53ef\u80fd\u4f1a\u53d1\u751f\u53d8\u5316\u751a\u81f3\u6574\u4f53\u79fb\u9664\u3002</li> <li>\u8be5\u529f\u80fd API \u4e0d\u4fdd\u8bc1\u7a33\u5b9a\uff0c\u53ef\u80fd\u4f1a\u88ab\u79fb\u9664\u3002</li> <li>\u8be5\u529f\u80fd\u7684\u793e\u533a\u652f\u6301\u4f18\u5148\u7ea7\u8f83\u4f4e\uff0c\u4e14\u65e0\u6cd5\u4fdd\u8bc1\u957f\u671f\u652f\u6301\u3002</li> <li>\u7531\u4e8e\u529f\u80fd\u7a33\u5b9a\u6027\u548c\u957f\u671f\u652f\u6301\u65e0\u6cd5\u4fdd\u8bc1\uff0c\u53ef\u4ee5\u8fdb\u884c\u6d4b\u8bd5\u9a8c\u8bc1\uff0c\u4f46\u4e0d\u63a8\u8350\u751f\u4ea7\u4f7f\u7528\u3002</li> </ul> <p>\u5bf9\u4e8e Beta \u529f\u80fd\uff1a</p> <ul> <li>\u8be5\u529f\u80fd\u6709\u90e8\u5206\u6587\u6863\u548c\u6d4b\u8bd5\uff0c\u4f46\u662f\u4e0d\u4fdd\u8bc1\u5b8c\u6574\u7684\u8986\u76d6\u3002</li> <li>\u8be5\u529f\u80fd\u672a\u6765\u53ef\u80fd\u53d1\u751f\u53d8\u5316\uff0c\u5347\u7ea7\u53ef\u80fd\u4f1a\u5f71\u54cd\u7f51\u7edc\uff0c\u4f46\u4e0d\u4f1a\u88ab\u6574\u4f53\u79fb\u9664\u3002</li> <li>\u8be5\u529f\u80fd API \u672a\u6765\u53ef\u80fd\u4f1a\u53d1\u751f\u53d8\u5316\uff0c\u5b57\u6bb5\u53ef\u80fd\u4f1a\u8fdb\u884c\u8c03\u6574\uff0c\u4f46\u4e0d\u4f1a\u6574\u4f53\u79fb\u9664\u3002</li> <li>\u8be5\u529f\u80fd\u4f1a\u5f97\u5230\u793e\u533a\u7684\u957f\u671f\u652f\u6301\u3002</li> <li>\u7531\u4e8e\u529f\u80fd\u4f1a\u5f97\u5230\u957f\u671f\u652f\u6301\uff0c\u53ef\u4ee5\u5728\u975e\u5173\u952e\u4e1a\u52a1\u4e0a\u8fdb\u884c\u4f7f\u7528\uff0c\u4f46\u662f\u7531\u4e8e\u529f\u80fd\u548c API \u5b58\u5728\u53d8\u5316\u7684\u53ef\u80fd\uff0c\u53ef\u80fd\u4f1a\u5728\u5347\u7ea7\u4e2d\u51fa\u73b0\u4e2d\u65ad\uff0c\u4e0d\u63a8\u8350\u5728\u5173\u952e\u751f\u4ea7\u4e1a\u52a1\u4e0a\u4f7f\u7528\u3002</li> </ul> <p>\u5bf9\u4e8e GA \u529f\u80fd\uff1a</p> <ul> <li>\u8be5\u529f\u80fd\u6709\u5b8c\u6574\u7684\u6587\u6863\u548c\u6d4b\u8bd5\u8986\u76d6\u3002</li> <li>\u8be5\u529f\u80fd\u4f1a\u4fdd\u6301\u7a33\u5b9a\uff0c\u5347\u7ea7\u4f1a\u4fdd\u8bc1\u5e73\u6ed1\u3002</li> <li>\u8be5\u529f\u80fd API \u4e0d\u4f1a\u53d1\u751f\u7834\u574f\u6027\u53d8\u5316\u3002</li> <li>\u8be5\u529f\u80fd\u4f1a\u5f97\u5230\u793e\u533a\u9ad8\u4f18\u5148\u7ea7\u652f\u6301\uff0c\u5e76\u4f1a\u4fdd\u8bc1\u957f\u671f\u652f\u6301\u3002</li> </ul>"},{"location":"reference/feature-stage/#_3","title":"\u6210\u719f\u5ea6\u5217\u8868","text":"<p>\u672c\u5217\u8868\u7edf\u8ba1\u4ece v1.8 \u7248\u672c\u4e2d\u5305\u542b\u7684\u529f\u80fd\u5bf9\u5e94\u6210\u719f\u5ea6\u3002</p> \u529f\u80fd \u9ed8\u8ba4\u5f00\u542f \u72b6\u6001 \u5f00\u59cb\uff08Since\uff09 \u7ed3\u675f\uff08Until\uff09 Namespaced Subnet true GA 1.8 \u5206\u5e03\u5f0f\u7f51\u5173 true GA 1.8 \u4e3b\u4ece\u6a21\u5f0f\u96c6\u4e2d\u5f0f\u7f51\u5173 true GA 1.8 ECMP \u6a21\u5f0f\u96c6\u4e2d\u5f0f\u7f51\u5173 false Beta 1.8 \u5b50\u7f51 ACL true Alpha 1.9 \u5b50\u7f51\u9694\u79bb (\u672a\u6765\u4f1a\u548c\u5b50\u7f51 ACL \u5408\u5e76) true Beta 1.8 Underlay \u5b50\u7f51 true GA 1.8 \u591a\u7f51\u5361\u7ba1\u7406 true Beta 1.8 \u5b50\u7f51 DHCP false Alpha 1.10 \u5b50\u7f51\u8bbe\u7f6e\u5916\u90e8\u7f51\u5173 false Alpha 1.8 \u4f7f\u7528 OVN-IC \u8fdb\u884c\u96c6\u7fa4\u4e92\u8054 false Beta 1.8 \u4f7f\u7528 Submariner \u8fdb\u884c\u96c6\u7fa4\u4e92\u8054 false Alpha 1.9 \u5b50\u7f51 VIP \u9884\u7559 true Alpha 1.10 \u521b\u5efa\u81ea\u5b9a\u4e49 VPC true Beta 1.8 \u81ea\u5b9a\u4e49 VPC \u6d6e\u52a8 IP/SNAT/DNAT true Alpha 1.10 \u81ea\u5b9a\u4e49 VPC \u9759\u6001\u8def\u7531 true Alpha 1.10 \u81ea\u5b9a\u4e49 VPC \u7b56\u7565\u8def\u7531 true Alpha 1.10 \u81ea\u5b9a\u4e49 VPC \u5b89\u5168\u7ec4 true Alpha 1.10 \u5bb9\u5668\u6700\u5927\u5e26\u5bbd QoS true GA 1.8 linux-netem QoS true Alpha 1.9 Prometheus \u96c6\u6210 false GA 1.8 Grafana \u96c6\u6210 false GA 1.8 \u53cc\u6808\u7f51\u7edc false GA 1.8 \u9ed8\u8ba4 VPC EIP/SNAT false Beta 1.8 \u6d41\u91cf\u955c\u50cf false GA 1.8 NetworkPolicy true Beta 1.8 Webhook false Alpha 1.10 \u6027\u80fd\u8c03\u4f18 false Beta 1.8 Overlay \u5b50\u7f51\u9759\u6001\u8def\u7531\u5bf9\u5916\u66b4\u9732 false Alpha 1.8 Overlay \u5b50\u7f51 BGP \u5bf9\u5916\u66b4\u9732 false Alpha 1.9 Cilium \u96c6\u6210 false Alpha 1.10 \u81ea\u5b9a\u4e49 VPC \u4e92\u8054 false Alpha 1.10 Mellanox Offload false Alpha 1.8 \u82af\u542f\u6e90 Offload false Alpha 1.10 Windows \u652f\u6301 false Alpha 1.10 DPDK \u652f\u6301 false Alpha 1.10 OpenStack \u96c6\u6210 false Alpha 1.9 \u5355\u4e2a Pod \u56fa\u5b9a IP/Mac true GA 1.8 Workload \u56fa\u5b9a IP true GA 1.8 StatefulSet \u56fa\u5b9a IP true GA 1.8 VM \u56fa\u5b9a IP false Beta 1.9 \u9ed8\u8ba4 VPC Load Balancer \u7c7b\u578b Service false Alpha 1.11 \u81ea\u5b9a\u4e49 VPC \u5185\u90e8\u8d1f\u8f7d\u5747\u8861 false Alpha 1.11 \u81ea\u5b9a\u4e49 VPC DNS false Alpha 1.11 Underlay \u548c Overlay \u4e92\u901a false Alpha 1.11 <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"reference/iptables-rules/","title":"Iptables \u89c4\u5219","text":"<p>Kube-OVN \u4f7f\u7528 ipset \u53ca iptables \u8f85\u52a9\u5b9e\u73b0\u9ed8\u8ba4 VPC \u4e0b\u5bb9\u5668\u7f51\u7edc\uff08Overlay\uff09\u7f51\u5173 NAT \u7684\u529f\u80fd\u3002</p> <p>\u4f7f\u7528\u7684 ipset \u5982\u4e0b\u8868\u6240\u793a\uff1a</p> \u540d\u79f0\uff08IPv4/IPv6\uff09 \u7c7b\u578b \u5b58\u50a8\u5bf9\u8c61 ovn40services/ovn60services hash:net Service \u7f51\u6bb5 ovn40subnets/ovn60subnets hash:net Overlay \u5b50\u7f51\u7f51\u6bb5\u4ee5\u53ca NodeLocal DNS IP \u5730\u5740 ovn40subnets-nat/ovn60subnets-nat hash:net \u5f00\u542f NatOutgoing \u7684 Overlay \u5b50\u7f51\u7f51\u6bb5 ovn40subnets-distributed-gw/ovn60subnets-distributed-gw hash:net \u5f00\u542f\u5206\u5e03\u5f0f\u7f51\u5173\u7684 Overlay \u5b50\u7f51\u7f51\u6bb5 ovn40other-node/ovn60other-node hash:net \u5176\u5b83\u8282\u70b9\u7684\u5185\u90e8 IP \u5730\u5740 ovn40local-pod-ip-nat/ovn60local-pod-ip-nat hash:ip \u5df2\u5f03\u7528 ovn40subnets-nat-policy hash:net \u914d\u7f6e\u4e86 natOutgoingPolicyRules \u7684\u6240\u6709\u5b50\u7f51\u7f51\u6bb5 ovn40natpr-418e79269dc5-dst hash:net natOutgoingPolicyRules \u4e2d rule \u5bf9\u5e94\u7684 dstIPs ovn40natpr-418e79269dc5-src hash:net natOutgoingPolicyRules \u4e2d rule \u5bf9\u5e94\u7684 srcIPs <p>\u4f7f\u7528\u7684 iptables \u89c4\u5219\uff08IPv4\uff09\u5982\u4e0b\u8868\u6240\u793a\uff1a</p> \u8868 \u94fe \u89c4\u5219 \u7528\u9014 \u5907\u6ce8 filter INPUT -m set --match-set ovn40services src -j ACCEPT \u5141\u8bb8 k8s Service \u548c Pod \u76f8\u5173\u6d41\u91cf\u901a\u8fc7 -- filter INPUT -m set --match-set ovn40services dst -j ACCEPT \u540c\u4e0a -- filter INPUT -m set --match-set ovn40subnets src -j ACCEPT \u540c\u4e0a -- filter INPUT -m set --match-set ovn40subnets dst -j ACCEPT \u540c\u4e0a -- filter FORWARD -m set --match-set ovn40services src -j ACCEPT \u540c\u4e0a -- filter FORWARD -m set --match-set ovn40services dst -j ACCEPT \u540c\u4e0a -- filter FORWARD -m set --match-set ovn40subnets src -j ACCEPT \u540c\u4e0a -- filter FORWARD -m set --match-set ovn40subnets dst -j ACCEPT \u540c\u4e0a -- filter FORWARD -s 10.16.0.0/16 -m comment --comment \"ovn-subnet-gateway,ovn-default\" \u7528\u4e8e\u8ba1\u6570\u4ece subnet \u8bbf\u95ee\u5916\u90e8\u7f51\u7edc\u7684\u62a5\u6587 10.16.0.0/16 \u4e3a subnet \u7684 cidr \uff0ccomment \u4e2d\u9017\u53f7\u524d\u9762\u7684 ovn-subnet-gateway \u7528\u4e8e\u6807\u8bc6\u8be5 iptables \u89c4\u5219\u7528\u4e8e subnet \u51fa\u5165\u7f51\u5173\u62a5\u6587\u8ba1\u6570\uff0c\u9017\u53f7\u540e\u9762 ovn-default \u662f\u8be5 subnet \u7684\u540d\u5b57 filter FORWARD -d 10.16.0.0/16 -m comment --comment \"ovn-subnet-gateway,ovn-default\" \u7528\u4e8e\u8ba1\u6570\u4ece\u5916\u90e8\u7f51\u7edc\u8bbf\u95ee subnet \u7684\u62a5\u6587 \u540c\u4e0a filter OUTPUT -p udp -m udp --dport 6081 -j MARK --set-xmark 0x0 \u6e05\u9664\u6d41\u91cf\u6807\u8bb0\uff0c\u907f\u514d\u6267\u884c SNAT UDP: bad checksum on VXLAN interface nat PREROUTING -m comment --comment \"kube-ovn prerouting rules\" -j OVN-PREROUTING \u8fdb\u5165 OVN-PREROUTING \u94fe\u5904\u7406 -- nat POSTROUTING -m comment --comment \"kube-ovn postrouting rules\" -j OVN-POSTROUTING \u8fdb\u5165 OVN-POSTROUTING \u94fe\u5904\u7406 -- nat OVN-PREROUTING -i ovn0 -m set --match-set ovn40subnets src -m set --match-set ovn40services dst -j MARK --set-xmark 0x4000/0x4000 \u4e3a Pod \u8bbf\u95ee Service \u6d41\u91cf\u6dfb\u52a0 masquerade \u6807\u8bb0 \u4f5c\u7528\u4e8e\u5173\u95ed\u5185\u7f6e LB \u7684\u573a\u666f nat OVN-PREROUTING -p tcp -m addrtype --dst-type LOCAL -m set --match-set KUBE-NODE-PORT-LOCAL-TCP dst -j MARK --set-xmark 0x80000/0x80000 \u4e3a ExternalTrafficPolicy \u4e3a Local \u7684 Service \u6d41\u91cf\uff08TCP\uff09\u6dfb\u52a0\u7279\u5b9a\u6807\u8bb0 \u4ec5 kube-proxy \u4f7f\u7528 ipvs \u6a21\u5f0f\u65f6\u5b58\u5728 nat OVN-PREROUTING -p udp -m addrtype --dst-type LOCAL -m set --match-set KUBE-NODE-PORT-LOCAL-UDP dst -j MARK --set-xmark 0x80000/0x80000 \u4e3a ExternalTrafficPolicy \u4e3a Local \u7684 Service \u6d41\u91cf\uff08UDP\uff09\u6dfb\u52a0\u7279\u5b9a\u6807\u8bb0 \u540c\u4e0a nat OVN-POSTROUTING -m set --match-set ovn40services src -m set --match-set ovn40subnets dst -m mark --mark 0x4000/0x4000 -j SNAT --to-source  \u5f53\u8282\u70b9\u901a\u8fc7 Service IP \u8bbf\u95ee Overlay Pod \u65f6\uff0c\u4fdd\u6301\u6e90 IP \u4e3a\u8282\u70b9 IP\u3002 \u4ec5 kube-proxy \u4f7f\u7528 ipvs \u6a21\u5f0f\u65f6\u751f\u6548 nat OVN-POSTROUTING -m mark --mark 0x4000/0x4000 -j MASQUERADE \u4e3a\u7279\u5b9a\u6807\u8bb0\u7684\u6d41\u91cf\u6267\u884c SNAT -- nat OVN-POSTROUTING -m set --match-set ovn40subnets src -m set --match-set ovn40subnets dst -j MASQUERADE \u4e3a\u901a\u8fc7\u8282\u70b9\u7684 Pod \u4e4b\u95f4\u7684 Service \u6d41\u91cf\u6267\u884c SNAT -- nat OVN-POSTROUTING -m mark --mark 0x80000/0x80000 -m set --match-set ovn40subnets-distributed-gw dst -j RETURN \u5bf9\u4e8e ExternalTrafficPolicy \u4e3a Local \u7684 Service \u6d41\u91cf\uff0c\u82e5 Endpoint \u4f7f\u7528\u5206\u5e03\u5f0f\u7f51\u5173\uff0c\u65e0\u9700\u6267\u884c SNAT -- nat OVN-POSTROUTING -m mark --mark 0x80000/0x80000 -j MASQUERADE \u5bf9\u4e8e ExternalTrafficPolicy \u4e3a Local \u7684 Service \u6d41\u91cf\uff0c\u82e5 Endpoint \u4f7f\u7528\u96c6\u4e2d\u5f0f\u7f51\u5173\uff0c\u6267\u884c SNAT -- nat OVN-POSTROUTING -p tcp -m tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN Pod IP \u5bf9\u5916\u66b4\u9732\u65f6\uff0c\u4e0d\u6267\u884c SNAT -- nat OVN-POSTROUTING -s 10.16.0.0/16 -m set ! --match-set ovn40subnets dst -j SNAT --to-source 192.168.0.101 Pod \u8bbf\u95ee\u96c6\u7fa4\u5916\u7f51\u7edc\u65f6\uff0c\u82e5\u5b50\u7f51\u5f00\u542f NatOutgoing \u4e14\u4f7f\u7528\u6307\u5b9a IP \u7684\u96c6\u4e2d\u5f0f\u7f51\u5173\uff0c\u6267\u884c SNAT 10.16.0.0/16 \u4e3a\u5b50\u7f51\u7f51\u6bb5\uff0c192.168.0.101 \u4e3a\u6307\u5b9a\u7684\u7f51\u5173\u8282\u70b9 IP nat OVN-POSTROUTING -m set --match-set ovn40subnets-nat src -m set ! --match-set ovn40subnets dst -j MASQUERADE Pod \u8bbf\u95ee\u96c6\u7fa4\u5916\u7f51\u7edc\u65f6\uff0c\u82e5\u5b50\u7f51\u5f00\u542f NatOutgoing\uff0c\u6267\u884c SNAT -- nat OVN-POSTROUTING -m set --match-set ovn40subnets-nat-policy src -m set ! --match-set ovn40subnets dst -j OVN-NAT-POLICY Pod \u8bbf\u95ee\u96c6\u7fa4\u5916\u7f51\u7edc\u65f6\uff0c\u82e5\u5b50\u7f51\u5f00\u542f natOutgoingPolicyRules\uff0c\u6307\u5b9a\u7b56\u7565\u7684\u62a5\u6587\u6267\u884c SNAT \u914d\u7f6e\u4e86 natOutgoingPolicyRules \u5b50\u7f51\u7684\u51fa\u5916\u7f51\u62a5\u6587\u7684\u8fdb\u5165\u94fe OVN-NAT-POLICY nat OVN-POSTROUTING -m mark --mark 0x90001/0x90001 -j MASQUERADE --random-fully \u540c\u4e0a \u4ece OVN-NAT-POLICY \u51fa\u6765\u540e\uff0c\u5982\u679c\u88ab\u6253\u4e0a tag 0x90001/0x90001 \u5c31\u4f1a\u505a SNAT nat OVN-POSTROUTING -m mark --mark 0x90002/0x90002 -j RETURN \u540c\u4e0a \u4ece OVN-NAT-POLICY \u51fa\u6765\u540e, \u5982\u679c\u88ab\u6253\u4e0a tag 0x90002/0x90002 \u4e0d\u4f1a\u505a SNAT nat OVN-NAT-POLICY -s 10.0.11.0/24 -m comment --comment natPolicySubnet-net1 -j OVN-NAT-PSUBNET-aa98851157c5 \u540c\u4e0a 10.0.11.0/24 \u8868\u793a\u5b50\u7f51 net1 \u7684 CIDR\uff0c  OVN-NAT-PSUBNET-aa98851157c5 \u8fd9\u6761\u94fe\u4e0b\u7684\u89c4\u5219\u5c31\u5bf9\u5e94\u8fd9\u4e2a\u5b50\u7f51\u7684 natOutgoingPolicyRules \u914d\u7f6e nat OVN-NAT-PSUBNET-xxxxxxxxxxxx -m set --match-set ovn40natpr-418e79269dc5-src src -m set --match-set ovn40natpr-418e79269dc5-dst dst -j MARK --set-xmark 0x90002/0x90002 \u540c\u4e0a 418e79269dc5 \u8868\u793a natOutgoingPolicyRules \u4e2d\u7684\u4e00\u6761\u89c4\u5219\u7684 ID\uff0c\u53ef\u4ee5\u901a\u8fc7 status.natOutgoingPolicyRules[index].RuleID \u67e5\u770b\u5230\uff0c \u8868\u793a srcIPs \u6ee1\u8db3 ovn40natpr-418e79269dc5-src\uff0c dstIPS \u6ee1\u8db3 ovn40natpr-418e79269dc5-dst \u4f1a\u6253\u4e0a tag 0x90002 mangle OVN-OUTPUT -d 10.241.39.2/32 -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x90003/0x90003 \u5c06 kubelet \u7684\u63a2\u6d4b\u6d41\u91cf\u52a0\u4e0a\u7279\u5b9a\u6807\u8bb0\u4ece\u800c\u5f15\u5165\u5230 tproxy mangle OVN-PREROUTING -d 10.241.39.2/32 -p tcp -m tcp --dport 80 -j TPROXY --on-port 8102 --on-ip 172.18.0.3 --tproxy-mark 0x90004/0x90004 \u5c06 kubelet \u7684\u63a2\u6d4b\u6d41\u91cf\u52a0\u4e0a\u7279\u5b9a\u6807\u8bb0\u4ece\u800c\u5f15\u5165\u5230 tproxy <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"reference/kube-ovn-api/","title":"Kube-OVN \u63a5\u53e3\u89c4\u8303","text":"<p>\u57fa\u4e8e Kube-OVN v1.12.0 \u7248\u672c\uff0c\u6574\u7406\u4e86 Kube-OVN \u652f\u6301\u7684 CRD \u8d44\u6e90\u5217\u8868\uff0c\u5217\u51fa CRD \u5b9a\u4e49\u5404\u5b57\u6bb5\u7684\u53d6\u503c\u7c7b\u578b\u548c\u542b\u4e49\uff0c\u4ee5\u4f9b\u53c2\u8003\u3002</p>"},{"location":"reference/kube-ovn-api/#condition","title":"\u901a\u7528\u7684 Condition \u5b9a\u4e49","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 type String \u72b6\u6001\u7c7b\u578b status String \u72b6\u6001\u503c\uff0c\u53d6\u503c\u4e3a <code>True</code>\uff0c<code>False</code> \u6216 <code>Unknown</code> reason String \u72b6\u6001\u53d8\u5316\u7684\u539f\u56e0 message String \u72b6\u6001\u53d8\u5316\u7684\u5177\u4f53\u4fe1\u606f lastUpdateTime Time \u4e0a\u6b21\u72b6\u6001\u66f4\u65b0\u65f6\u95f4 lastTransitionTime Time \u4e0a\u6b21\u72b6\u6001\u7c7b\u578b\u53d1\u751f\u53d8\u5316\u7684\u65f6\u95f4 <p>\u5728\u5404 CRD \u7684\u5b9a\u4e49\u4e2d\uff0cStatus \u4e2d\u7684 Condition \u5b57\u6bb5\uff0c\u90fd\u9075\u5faa\u4e0a\u8ff0\u683c\u5f0f\uff0c\u56e0\u6b64\u63d0\u524d\u8fdb\u884c\u8bf4\u660e\u3002</p>"},{"location":"reference/kube-ovn-api/#subnet","title":"Subnet \u5b9a\u4e49","text":""},{"location":"reference/kube-ovn-api/#subnet_1","title":"Subnet","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>Subnet</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec SubnetSpec Subnet \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status SubnetStatus Subnet \u72b6\u6001\u4fe1\u606f\u5b57\u6bb5"},{"location":"reference/kube-ovn-api/#subnetspec","title":"SubnetSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 default Bool \u8be5\u5b50\u7f51\u662f\u5426\u4e3a\u9ed8\u8ba4\u5b50\u7f51 vpc String \u5b50\u7f51\u6240\u5c5e Vpc\uff0c\u9ed8\u8ba4\u4e3a ovn-cluster protocol String IP \u534f\u8bae\uff0c\u53d6\u503c\u53ef\u4ee5\u4e3a <code>IPv4</code>\uff0c<code>IPv6</code> \u6216 <code>Dual</code> namespaces []String \u8be5\u5b50\u7f51\u6240\u7ed1\u5b9a\u7684 namespace \u5217\u8868 cidrBlock String \u5b50\u7f51\u7684\u7f51\u6bb5\u8303\u56f4\uff0c\u5982 10.16.0.0/16 gateway String \u5b50\u7f51\u7f51\u5173\u5730\u5740\uff0c\u9ed8\u8ba4\u4e3a\u8be5\u5b50\u7f51 CIDRBlock \u4e0b\u7684\u7b2c\u4e00\u4e2a\u53ef\u7528\u5730\u5740 excludeIps []String \u8be5\u5b50\u7f51\u4e0b\u4e0d\u4f1a\u88ab\u81ea\u52a8\u5206\u914d\u7684\u5730\u5740\u8303\u56f4 provider String \u9ed8\u8ba4\u4e3a ovn\u3002\u591a\u7f51\u5361\u60c5\u51b5\u4e0b\u53ef\u4ee5\u914d\u7f6e\u53d6\u503c\u4e3a NetworkAttachmentDefinition \u7684 .\uff0cKube-OVN \u5c06\u4f1a\u4f7f\u7528\u8fd9\u4e9b\u4fe1\u606f\u627e\u5230\u5bf9\u5e94\u7684 Subnet \u8d44\u6e90 gatewayType String Overlay \u6a21\u5f0f\u4e0b\u7684\u7f51\u5173\u7c7b\u578b\uff0c\u53d6\u503c\u53ef\u4ee5\u4e3a <code>distributed</code> \u6216 <code>centralized</code> gatewayNode String \u5f53\u7f51\u5173\u6a21\u5f0f\u4e3a centralized \u65f6\u7684\u7f51\u5173\u8282\u70b9\uff0c\u53ef\u4ee5\u4e3a\u9017\u53f7\u5206\u9694\u7684\u591a\u4e2a\u8282\u70b9 natOutgoing Bool \u51fa\u7f51\u6d41\u91cf\u662f\u5426\u8fdb\u884c NAT\u3002\u8be5\u53c2\u6570\u548c <code>externalEgressGateway</code> \u53c2\u6570\u4e0d\u80fd\u540c\u65f6\u8bbe\u7f6e\u3002 externalEgressGateway String \u5916\u90e8\u7f51\u5173\u5730\u5740\u3002\u9700\u8981\u548c\u5b50\u7f51\u7f51\u5173\u8282\u70b9\u5728\u540c\u4e00\u4e2a\u4e8c\u5c42\u53ef\u8fbe\u57df\uff0c\u8be5\u53c2\u6570\u548c <code>natOutgoing</code> \u53c2\u6570\u4e0d\u80fd\u540c\u65f6\u8bbe\u7f6e policyRoutingPriority Uint32 \u7b56\u7565\u8def\u7531\u4f18\u5148\u7ea7\u3002\u6dfb\u52a0\u7b56\u7565\u8def\u7531\u4f7f\u7528\u53c2\u6570\uff0c\u63a7\u5236\u6d41\u91cf\u7ecf\u5b50\u7f51\u7f51\u5173\u4e4b\u540e\uff0c\u8f6c\u53d1\u5230\u5916\u90e8\u7f51\u5173\u5730\u5740 policyRoutingTableID Uint32 \u4f7f\u7528\u7684\u672c\u5730\u7b56\u7565\u8def\u7531\u8868\u7684 TableID\uff0c\u6bcf\u4e2a\u5b50\u7f51\u5747\u9700\u4e0d\u540c\u4ee5\u907f\u514d\u51b2\u7a81 private Bool \u6807\u8bc6\u8be5\u5b50\u7f51\u662f\u5426\u4e3a\u79c1\u6709\u5b50\u7f51\uff0c\u79c1\u6709\u5b50\u7f51\u9ed8\u8ba4\u62d2\u7edd\u5b50\u7f51\u5916\u7684\u5730\u5740\u8bbf\u95ee allowSubnets []String \u5b50\u7f51\u4e3a\u79c1\u6709\u5b50\u7f51\u7684\u60c5\u51b5\u4e0b\uff0c\u5141\u8bb8\u8bbf\u95ee\u8be5\u5b50\u7f51\u5730\u5740\u7684\u96c6\u5408 vlan String \u5b50\u7f51\u7ed1\u5b9a\u7684 Vlan \u540d\u79f0 vips []String \u5b50\u7f51\u4e0b virtual \u7c7b\u578b lsp \u7684 virtual-ip \u53c2\u6570\u4fe1\u606f logicalGateway Bool \u662f\u5426\u542f\u7528\u903b\u8f91\u7f51\u5173 disableGatewayCheck Bool \u521b\u5efa Pod \u65f6\u662f\u5426\u8df3\u8fc7\u7f51\u5173\u8054\u901a\u6027\u68c0\u67e5 disableInterConnection Bool \u63a7\u5236\u662f\u5426\u5f00\u542f\u5b50\u7f51\u8de8\u96c6\u7fa4\u4e92\u8054 enableDHCP Bool \u63a7\u5236\u662f\u5426\u914d\u7f6e\u5b50\u7f51\u4e0b lsp \u7684 dhcp \u914d\u7f6e\u9009\u9879 dhcpV4Options String \u5b50\u7f51\u4e0b lsp dhcpv4_options \u5173\u8054\u7684 DHCP_Options \u8bb0\u5f55 dhcpV6Options String \u5b50\u7f51\u4e0b lsp dhcpv6_options \u5173\u8054\u7684 DHCP_Options \u8bb0\u5f55 enableIPv6RA Bool \u63a7\u5236\u5b50\u7f51\u8fde\u63a5\u8def\u7531\u5668\u7684 lrp \u7aef\u53e3\uff0c\u662f\u5426\u914d\u7f6e ipv6_ra_configs \u53c2\u6570 ipv6RAConfigs String \u5b50\u7f51\u8fde\u63a5\u8def\u7531\u5668\u7684 lrp \u7aef\u53e3\uff0cipv6_ra_configs \u53c2\u6570\u914d\u7f6e\u4fe1\u606f acls []Acl \u5b50\u7f51\u5bf9\u5e94 logical-switch \u5173\u8054\u7684 acls \u8bb0\u5f55 u2oInterconnection Bool \u662f\u5426\u5f00\u542f Overlay/Underlay \u7684\u4e92\u8054\u6a21\u5f0f enableLb *Bool \u63a7\u5236\u5b50\u7f51\u5bf9\u5e94\u7684 logical-switch \u662f\u5426\u5173\u8054 load-balancer \u8bb0\u5f55 enableEcmp Bool \u96c6\u4e2d\u5f0f\u7f51\u5173\uff0c\u662f\u5426\u5f00\u542f ECMP \u8def\u7531"},{"location":"reference/kube-ovn-api/#acl","title":"Acl","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 direction String Acl \u9650\u5236\u65b9\u5411\uff0c\u53d6\u503c\u4e3a <code>from-lport</code> \u6216\u8005 <code>to-lport</code> priority Int Acl \u4f18\u5148\u7ea7\uff0c\u53d6\u503c\u8303\u56f4 0 \u5230 32767 match String Acl \u89c4\u5219\u5339\u914d\u8868\u8fbe\u5f0f action String Acl \u89c4\u5219\u52a8\u4f5c\uff0c\u53d6\u503c\u4e3a <code>allow-related</code>, <code>allow-stateless</code>, <code>allow</code>, <code>drop</code>, <code>reject</code> \u5176\u4e2d\u4e00\u4e2a"},{"location":"reference/kube-ovn-api/#subnetstatus","title":"SubnetStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 conditions []SubnetCondition \u5b50\u7f51\u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49 v4AvailableIPs Float64 \u5b50\u7f51\u73b0\u5728\u53ef\u7528\u7684 IPv4 IP \u5730\u5740\u6570\u91cf v4availableIPrange String \u5b50\u7f51\u73b0\u5728\u53ef\u7528\u7684 IPv4 \u5730\u5740\u8303\u56f4 v4UsingIPs Float64 \u5b50\u7f51\u73b0\u5728\u5df2\u7528\u7684 IPv4 IP \u5730\u5740\u6570\u91cf v4usingIPrange String \u5b50\u7f51\u73b0\u5728\u5df2\u7528\u7684 IPv4 \u5730\u5740\u8303\u56f4 v6AvailableIPs Float64 \u5b50\u7f51\u73b0\u5728\u53ef\u7528\u7684 IPv6 IP \u5730\u5740\u6570\u91cf v6availableIPrange String \u5b50\u7f51\u73b0\u5728\u53ef\u7528\u7684 IPv6 \u5730\u5740\u8303\u56f4 v6UsingIPs Float64 \u5b50\u7f51\u73b0\u5728\u5df2\u7528\u7684 IPv6 IP \u5730\u5740\u6570\u91cf v6usingIPrange String \u5b50\u7f51\u73b0\u5728\u5df2\u7528\u7684 IPv6 \u5730\u5740\u8303\u56f4 sctivateGateway String \u96c6\u4e2d\u5f0f\u5b50\u7f51\uff0c\u4e3b\u5907\u6a21\u5f0f\u4e0b\u5f53\u524d\u6b63\u5728\u5de5\u4f5c\u7684\u7f51\u5173\u8282\u70b9 dhcpV4OptionsUUID String \u5b50\u7f51\u4e0b lsp dhcpv4_options \u5173\u8054\u7684 DHCP_Options \u8bb0\u5f55\u6807\u8bc6 dhcpV6OptionsUUID String \u5b50\u7f51\u4e0b lsp dhcpv6_options \u5173\u8054\u7684 DHCP_Options \u8bb0\u5f55\u6807\u8bc6 u2oInterconnectionIP String \u5f00\u542f Overlay/Underlay \u4e92\u8054\u6a21\u5f0f\u540e\uff0c\u6240\u5360\u7528\u7684\u7528\u4e8e\u4e92\u8054\u7684 IP \u5730\u5740"},{"location":"reference/kube-ovn-api/#ip","title":"IP \u5b9a\u4e49","text":""},{"location":"reference/kube-ovn-api/#ip_1","title":"IP","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>IP</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec IPSpec IP \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5"},{"location":"reference/kube-ovn-api/#ipsepc","title":"IPSepc","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 podName String \u7ed1\u5b9a Pod \u540d\u79f0 namespace String \u7ed1\u5b9a Pod \u6240\u5728 Namespace \u540d\u79f0 subnet String IP \u6240\u5c5e Subnet attachSubnets []String \u8be5\u4e3b IP \u4e0b\u5176\u4ed6\u9644\u5c5e\u5b50\u7f51\u540d\u79f0\uff08\u5b57\u6bb5\u5e9f\u5f03\u4e0d\u518d\u4f7f\u7528\uff09 nodeName String \u7ed1\u5b9a Pod \u6240\u5728\u7684\u8282\u70b9\u540d\u79f0 ipAddress String IP \u5730\u5740\uff0c\u53cc\u6808\u60c5\u51b5\u4e0b\u4e3a <code>v4IP,v6IP</code> \u683c\u5f0f v4IPAddress String IPv4 IP \u5730\u5740 v6IPAddress String IPv6 IP \u5730\u5740 attachIPs []String \u8be5\u4e3b IP \u4e0b\u5176\u4ed6\u9644\u5c5e IP \u5730\u5740\uff08\u5b57\u6bb5\u5e9f\u5f03\u4e0d\u518d\u4f7f\u7528\uff09 macAddress String \u7ed1\u5b9a Pod \u7684 Mac \u5730\u5740 attachMacs []String \u8be5\u4e3b IP \u4e0b\u5176\u4ed6\u9644\u5c5e Mac \u5730\u5740\uff08\u5b57\u6bb5\u5e9f\u5f03\u4e0d\u518d\u4f7f\u7528\uff09 containerID String \u7ed1\u5b9a Pod \u5bf9\u5e94\u7684 Container ID podType String \u7279\u6b8a\u5de5\u4f5c\u8d1f\u8f7d Pod\uff0c\u53ef\u4e3a <code>StatefulSet</code>\uff0c<code>VirtualMachine</code> \u6216\u7a7a"},{"location":"reference/kube-ovn-api/#underlay","title":"Underlay \u914d\u7f6e","text":""},{"location":"reference/kube-ovn-api/#vlan","title":"Vlan","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>Vlan</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec VlanSpec Vlan \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status VlanStatus Vlan \u72b6\u6001\u4fe1\u606f\u5b57\u6bb5"},{"location":"reference/kube-ovn-api/#vlanspec","title":"VlanSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 id Int Vlan tag \u53f7\uff0c\u53d6\u503c\u8303\u56f4\u4e3a 0~4096 provider String Vlan \u7ed1\u5b9a\u7684 ProviderNetwork \u540d\u79f0"},{"location":"reference/kube-ovn-api/#vlanstatus","title":"VlanStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 subnets []String Vlan \u7ed1\u5b9a\u7684\u5b50\u7f51\u5217\u8868 conditions []VlanCondition Vlan \u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49"},{"location":"reference/kube-ovn-api/#providernetwork","title":"ProviderNetwork","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>ProviderNetwork</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec ProviderNetworkSpec ProviderNetwork \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status ProviderNetworkStatus ProviderNetwork \u72b6\u6001\u4fe1\u606f\u5b57\u6bb5"},{"location":"reference/kube-ovn-api/#providernetworkspec","title":"ProviderNetworkSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 defaultInterface String \u8be5\u6865\u63a5\u7f51\u7edc\u9ed8\u8ba4\u4f7f\u7528\u7684\u7f51\u5361\u63a5\u53e3\u540d\u79f0 customInterfaces []CustomInterface \u8be5\u6865\u63a5\u7f51\u7edc\u7279\u6b8a\u4f7f\u7528\u7684\u7f51\u5361\u914d\u7f6e excludeNodes []String \u8be5\u6865\u63a5\u7f51\u7edc\u4e0d\u4f1a\u7ed1\u5b9a\u7684\u8282\u70b9\u540d\u79f0 exchangeLinkName Bool \u662f\u5426\u4ea4\u6362\u6865\u63a5\u7f51\u5361\u548c\u5bf9\u5e94 OVS \u7f51\u6865\u540d\u79f0"},{"location":"reference/kube-ovn-api/#custominterface","title":"CustomInterface","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 interface String Underlay \u4f7f\u7528\u7f51\u5361\u63a5\u53e3\u540d\u79f0 nodes []String \u4f7f\u7528\u81ea\u5b9a\u4e49\u7f51\u5361\u63a5\u53e3\u7684\u8282\u70b9\u5217\u8868"},{"location":"reference/kube-ovn-api/#providernetworkstatus","title":"ProviderNetworkStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 ready Bool \u5f53\u524d\u6865\u63a5\u7f51\u7edc\u662f\u5426\u8fdb\u5165\u5c31\u7eea\u72b6\u6001 readyNodes []String \u6865\u63a5\u7f51\u7edc\u8fdb\u5165\u5c31\u7eea\u72b6\u6001\u7684\u8282\u70b9\u540d\u79f0 notReadyNodes []String \u6865\u63a5\u7f51\u7edc\u672a\u8fdb\u5165\u5c31\u7eea\u72b6\u6001\u7684\u8282\u70b9\u540d\u79f0 vlans []String \u6865\u63a5\u7f51\u7edc\u7ed1\u5b9a\u7684 Vlan \u540d\u79f0 conditions []ProviderNetworkCondition ProviderNetwork \u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49"},{"location":"reference/kube-ovn-api/#vpc","title":"Vpc \u5b9a\u4e49","text":""},{"location":"reference/kube-ovn-api/#vpc_1","title":"Vpc","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>Vpc</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec VpcSpec Vpc \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status VpcStatus Vpc \u72b6\u6001\u4fe1\u606f\u5b57\u6bb5"},{"location":"reference/kube-ovn-api/#vpcspec","title":"VpcSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 namespaces []String Vpc \u7ed1\u5b9a\u7684\u547d\u540d\u7a7a\u95f4\u5217\u8868 staticRoutes []*StaticRoute Vpc \u4e0b\u914d\u7f6e\u7684\u9759\u6001\u8def\u7531\u4fe1\u606f policyRoutes []*PolicyRoute Vpc \u4e0b\u914d\u7f6e\u7684\u7b56\u7565\u8def\u7531\u4fe1\u606f vpcPeerings []*VpcPeering Vpc \u4e92\u8054\u4fe1\u606f enableExternal Bool Vpc \u662f\u5426\u8fde\u63a5\u5230\u5916\u90e8\u4ea4\u6362\u673a"},{"location":"reference/kube-ovn-api/#staticroute","title":"StaticRoute","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 policy String \u8def\u7531\u7b56\u7565\uff0c\u53d6\u503c\u4e3a <code>policySrc</code> \u6216\u8005 <code>policyDst</code> cidr String \u8def\u7531 Cidr \u7f51\u6bb5 nextHopIP String \u8def\u7531\u4e0b\u4e00\u8df3\u4fe1\u606f"},{"location":"reference/kube-ovn-api/#policyroute","title":"PolicyRoute","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 priority Int32 \u7b56\u7565\u8def\u7531\u4f18\u5148\u7ea7 match String \u7b56\u7565\u8def\u7531\u5339\u914d\u6761\u4ef6 action String \u7b56\u7565\u8def\u7531\u52a8\u4f5c\uff0c\u53d6\u503c\u4e3a <code>allow</code>\u3001<code>drop</code> \u6216\u8005 <code>reroute</code> nextHopIP String \u7b56\u7565\u8def\u7531\u4e0b\u4e00\u8df3\u4fe1\u606f\uff0cECMP \u8def\u7531\u60c5\u51b5\u4e0b\u4e0b\u4e00\u8df3\u5730\u5740\u4f7f\u7528\u9017\u53f7\u9694\u5f00"},{"location":"reference/kube-ovn-api/#vpcpeering","title":"VpcPeering","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 remoteVpc String Vpc \u4e92\u8054\u5bf9\u7aef Vpc \u540d\u79f0 localConnectIP String Vpc \u4e92\u8054\u672c\u7aef IP \u5730\u5740"},{"location":"reference/kube-ovn-api/#vpcstatus","title":"VpcStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 conditions []VpcCondition Vpc \u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49 standby Bool \u6807\u8bc6 Vpc \u662f\u5426\u521b\u5efa\u5b8c\u6210\uff0cVpc \u4e0b\u7684 Subnet \u9700\u8981\u7b49 Vpc \u521b\u5efa\u5b8c\u6210\u8f6c\u6362\u518d\u7ee7\u7eed\u5904\u7406 default Bool \u662f\u5426\u662f\u9ed8\u8ba4 Vpc defaultLogicalSwitch String Vpc \u4e0b\u7684\u9ed8\u8ba4\u5b50\u7f51 router String Vpc \u5bf9\u5e94\u7684 logical-router \u540d\u79f0 tcpLoadBalancer String Vpc \u4e0b\u7684 TCP LB \u4fe1\u606f udpLoadBalancer String Vpc \u4e0b\u7684 UDP LB \u4fe1\u606f tcpSessionLoadBalancer String Vpc \u4e0b\u7684 TCP \u4f1a\u8bdd\u4fdd\u6301 LB \u4fe1\u606f udpSessionLoadBalancer String Vpc \u4e0b\u7684 UDP \u4f1a\u8bdd\u4fdd\u6301 LB \u4fe1\u606f subnets []String Vpc \u4e0b\u7684\u5b50\u7f51\u5217\u8868 vpcPeerings []String Vpc \u4e92\u8054\u7684\u5bf9\u7aef Vpc \u5217\u8868 enableExternal Bool Vpc \u662f\u5426\u8fde\u63a5\u5230\u5916\u90e8\u4ea4\u6362\u673a"},{"location":"reference/kube-ovn-api/#vpcnatgateway","title":"VpcNatGateway","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>VpcNatGateway</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec VpcNatSpec Vpc \u7f51\u5173\u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5"},{"location":"reference/kube-ovn-api/#vpcnatspec","title":"VpcNatSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 vpc String Vpc \u7f51\u5173 Pod \u6240\u5728\u7684 Vpc \u540d\u79f0 subnet String Vpc \u7f51\u5173 Pod \u6240\u5c5e\u7684\u5b50\u7f51\u540d\u79f0 lanIp String Vpc \u7f51\u5173 Pod \u6307\u5b9a\u5206\u914d\u7684 IP \u5730\u5740 selector []String \u6807\u51c6 Kubernetes Selector \u5339\u914d\u4fe1\u606f tolerations []VpcNatToleration \u6807\u51c6 Kubernetes \u5bb9\u5fcd\u4fe1\u606f"},{"location":"reference/kube-ovn-api/#vpcnattoleration","title":"VpcNatToleration","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 key String \u5bb9\u5fcd\u6c61\u70b9\u7684 key \u4fe1\u606f operator String \u53d6\u503c\u4e3a <code>Exists</code> \u6216\u8005 <code>Equal</code> value String \u5bb9\u5fcd\u6c61\u70b9\u7684 value \u4fe1\u606f effect String \u5bb9\u5fcd\u6c61\u70b9\u7684\u4f5c\u7528\u6548\u679c\uff0c\u53d6\u503c\u4e3a <code>NoExecute</code> \u3001<code>NoSchedule</code> \u6216\u8005 <code>PreferNoSchedule</code> tolerationSeconds Int64 \u6dfb\u52a0\u6c61\u70b9\u540e\uff0cPod \u8fd8\u80fd\u7ee7\u7eed\u5728\u8282\u70b9\u4e0a\u8fd0\u884c\u7684\u65f6\u95f4 <p>\u4ee5\u4e0a\u5bb9\u5fcd\u5b57\u6bb5\u7684\u542b\u4e49\uff0c\u53ef\u4ee5\u53c2\u8003 Kubernetes \u5b98\u65b9\u6587\u6863 \u6c61\u70b9\u548c\u5bb9\u5fcd\u5ea6\u3002</p>"},{"location":"reference/kube-ovn-api/#iptableseip","title":"IptablesEIP","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>IptablesEIP</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec IptablesEipSpec Vpc \u7f51\u5173\u4f7f\u7528\u7684 IptablesEIP \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status IptablesEipStatus Vpc \u7f51\u5173\u4f7f\u7528\u7684 IptablesEIP \u72b6\u6001\u4fe1\u606f"},{"location":"reference/kube-ovn-api/#iptableseipspec","title":"IptablesEipSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 v4ip String IptablesEIP v4 \u5730\u5740 v6ip String IptablesEIP v6 \u5730\u5740 macAddress String IptablesEIP crd \u8bb0\u5f55\u5206\u914d\u7684 mac \u5730\u5740\uff0c\u6ca1\u6709\u5b9e\u9645\u4f7f\u7528 natGwDp String Vpc \u7f51\u5173\u540d\u79f0"},{"location":"reference/kube-ovn-api/#iptableseipstatus","title":"IptablesEipStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 ready Bool IptablesEIP \u662f\u5426\u914d\u7f6e\u5b8c\u6210 ip String IptablesEIP \u4f7f\u7528\u7684 IP \u5730\u5740\uff0c\u76ee\u524d\u53ea\u652f\u6301\u4e86 IPv4 \u5730\u5740 redo String IptablesEIP crd \u521b\u5efa\u6216\u8005\u66f4\u65b0\u65f6\u95f4 nat String IptablesEIP \u7684\u4f7f\u7528\u7c7b\u578b\uff0c\u53d6\u503c\u4e3a <code>fip</code>\u3001<code>snat</code> \u6216\u8005 <code>dnat</code> conditions []IptablesEIPCondition IptablesEIP \u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49"},{"location":"reference/kube-ovn-api/#iptablesfiprule","title":"IptablesFIPRule","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>IptablesFIPRule</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec IptablesFIPRuleSpec Vpc \u7f51\u5173\u4f7f\u7528\u7684 IptablesFIPRule \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status IptablesFIPRuleStatus Vpc \u7f51\u5173\u4f7f\u7528\u7684 IptablesFIPRule \u72b6\u6001\u4fe1\u606f"},{"location":"reference/kube-ovn-api/#iptablesfiprulespec","title":"IptablesFIPRuleSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 eip String IptablesFIPRule \u4f7f\u7528\u7684 IptablesEIP \u540d\u79f0 internalIp String IptablesFIPRule \u5bf9\u5e94\u7684\u5185\u90e8\u7684 IP \u5730\u5740"},{"location":"reference/kube-ovn-api/#iptablesfiprulestatus","title":"IptablesFIPRuleStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 ready Bool IptablesFIPRule \u662f\u5426\u914d\u7f6e\u5b8c\u6210 v4ip String IptablesEIP \u4f7f\u7528\u7684 v4 IP \u5730\u5740 v6ip String IptablesEIP \u4f7f\u7528\u7684 v6 IP \u5730\u5740 natGwDp String Vpc \u7f51\u5173\u540d\u79f0 redo String IptablesFIPRule crd \u521b\u5efa\u6216\u8005\u66f4\u65b0\u65f6\u95f4 conditions []IptablesFIPRuleCondition IptablesFIPRule \u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49"},{"location":"reference/kube-ovn-api/#iptablessnatrule","title":"IptablesSnatRule","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>IptablesSnatRule</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec IptablesSnatRuleSpec Vpc \u7f51\u5173\u4f7f\u7528\u7684 IptablesSnatRule \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status IptablesSnatRuleStatus Vpc \u7f51\u5173\u4f7f\u7528\u7684 IptablesSnatRule \u72b6\u6001\u4fe1\u606f"},{"location":"reference/kube-ovn-api/#iptablessnatrulespec","title":"IptablesSnatRuleSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 eip String IptablesSnatRule \u4f7f\u7528\u7684 IptablesEIP \u540d\u79f0 internalIp String IptablesSnatRule \u5bf9\u5e94\u7684\u5185\u90e8\u7684 IP \u5730\u5740"},{"location":"reference/kube-ovn-api/#iptablessnatrulestatus","title":"IptablesSnatRuleStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 ready Bool IptablesSnatRule \u662f\u5426\u914d\u7f6e\u5b8c\u6210 v4ip String IptablesSnatRule \u4f7f\u7528\u7684 v4 IP \u5730\u5740 v6ip String IptablesSnatRule \u4f7f\u7528\u7684 v6 IP \u5730\u5740 natGwDp String Vpc \u7f51\u5173\u540d\u79f0 redo String IptablesSnatRule crd \u521b\u5efa\u6216\u8005\u66f4\u65b0\u65f6\u95f4 conditions []IptablesSnatRuleCondition IptablesSnatRule \u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49"},{"location":"reference/kube-ovn-api/#iptablesdnatrule","title":"IptablesDnatRule","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>IptablesDnatRule</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec IptablesDnatRuleSpec Vpc \u7f51\u5173\u4f7f\u7528\u7684 IptablesDnatRule \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status IptablesDnatRuleStatus Vpc \u7f51\u5173\u4f7f\u7528\u7684 IptablesDnatRule \u72b6\u6001\u4fe1\u606f"},{"location":"reference/kube-ovn-api/#iptablesdnatrulespec","title":"IptablesDnatRuleSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 eip Sting Vpc \u7f51\u5173\u914d\u7f6e IptablesDnatRule \u4f7f\u7528\u7684 IptablesEIP \u540d\u79f0 externalPort Sting Vpc \u7f51\u5173\u914d\u7f6e IptablesDnatRule \u4f7f\u7528\u7684\u5916\u90e8\u7aef\u53e3 protocol Sting Vpc \u7f51\u5173\u914d\u7f6e IptablesDnatRule \u7684\u534f\u8bae\u7c7b\u578b internalIp Sting Vpc \u7f51\u5173\u914d\u7f6e IptablesDnatRule \u4f7f\u7528\u7684\u5185\u90e8 IP \u5730\u5740 internalPort Sting Vpc \u7f51\u5173\u914d\u7f6e IptablesDnatRule \u4f7f\u7528\u7684\u5185\u90e8\u7aef\u53e3"},{"location":"reference/kube-ovn-api/#iptablesdnatrulestatus","title":"IptablesDnatRuleStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 ready Bool IptablesDnatRule \u662f\u5426\u914d\u7f6e\u5b8c\u6210 v4ip String IptablesDnatRule \u4f7f\u7528\u7684 v4 IP \u5730\u5740 v6ip String IptablesDnatRule \u4f7f\u7528\u7684 v6 IP \u5730\u5740 natGwDp String Vpc \u7f51\u5173\u540d\u79f0 redo String IptablesDnatRule crd \u521b\u5efa\u6216\u8005\u66f4\u65b0\u65f6\u95f4 conditions []IptablesDnatRuleCondition IptablesDnatRule \u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49"},{"location":"reference/kube-ovn-api/#vpcdns","title":"VpcDns","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>VpcDns</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec VpcDnsSpec VpcDns \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status VpcDnsStatus VpcDns \u72b6\u6001\u4fe1\u606f"},{"location":"reference/kube-ovn-api/#vpcdnsspec","title":"VpcDnsSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 vpc String VpcDns \u6240\u5728\u7684 Vpc \u540d\u79f0 subnet String VpcDns Pod \u5206\u914d\u5730\u5740\u7684 Subnet \u540d\u79f0"},{"location":"reference/kube-ovn-api/#vpcdnsstatus","title":"VpcDnsStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 conditions []VpcDnsCondition VpcDns \u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49 active Bool VpcDns \u662f\u5426\u6b63\u5728\u4f7f\u7528 <p>VpcDns \u7684\u8be6\u7ec6\u4f7f\u7528\u6587\u6863\uff0c\u53ef\u4ee5\u53c2\u8003 \u81ea\u5b9a\u4e49 VPC DNS\u3002</p>"},{"location":"reference/kube-ovn-api/#switchlbrule","title":"SwitchLBRule","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>SwitchLBRule</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec SwitchLBRuleSpec SwitchLBRule \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status SwitchLBRuleStatus SwitchLBRule \u72b6\u6001\u4fe1\u606f"},{"location":"reference/kube-ovn-api/#switchlbrulespec","title":"SwitchLBRuleSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 vip String SwitchLBRule \u914d\u7f6e\u7684 vip \u5730\u5740 namespace String SwitchLBRule \u7684\u547d\u540d\u7a7a\u95f4 selector []String \u6807\u51c6 Kubernetes Selector \u5339\u914d\u4fe1\u606f sessionAffinity String \u6807\u51c6 Kubernetes Service \u4e2d sessionAffinity \u53d6\u503c ports []SlrPort SwitchLBRule \u7aef\u53e3\u5217\u8868 <p>SwitchLBRule \u7684\u8be6\u7ec6\u914d\u7f6e\u4fe1\u606f\uff0c\u53ef\u4ee5\u53c2\u8003 \u81ea\u5b9a\u4e49 VPC \u5185\u90e8\u8d1f\u8f7d\u5747\u8861\u3002</p>"},{"location":"reference/kube-ovn-api/#slrport","title":"SlrPort","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 name String \u7aef\u53e3\u540d\u79f0 port Int32 \u7aef\u53e3\u53f7 targetPort Int32 \u76ee\u6807\u7aef\u53e3\u53f7 protocol String \u534f\u8bae\u7c7b\u578b"},{"location":"reference/kube-ovn-api/#switchlbrulestatus","title":"SwitchLBRuleStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 conditions []SwitchLBRuleCondition SwitchLBRule \u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49 ports String SwitchLBRule \u7aef\u53e3\u4fe1\u606f service String SwitchLBRule \u63d0\u4f9b\u670d\u52a1\u7684 service \u540d\u79f0"},{"location":"reference/kube-ovn-api/#vip","title":"\u5b89\u5168\u7ec4\u4e0e Vip","text":""},{"location":"reference/kube-ovn-api/#securitygroup","title":"SecurityGroup","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>SecurityGroup</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec SecurityGroupSpec \u5b89\u5168\u7ec4\u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status SecurityGroupStatus \u5b89\u5168\u7ec4\u72b6\u6001\u4fe1\u606f"},{"location":"reference/kube-ovn-api/#securitygroupspec","title":"SecurityGroupSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 ingressRules []*SgRule \u5165\u65b9\u5411\u5b89\u5168\u7ec4\u89c4\u5219 egressRules []*SgRule \u51fa\u65b9\u5411\u5b89\u5168\u7ec4\u89c4\u5219 allowSameGroupTraffic Bool \u540c\u4e00\u5b89\u5168\u7ec4\u5185\u7684 lsp \u662f\u5426\u53ef\u4ee5\u4e92\u901a\uff0c\u4ee5\u53ca\u6d41\u91cf\u89c4\u5219\u662f\u5426\u9700\u8981\u66f4\u65b0"},{"location":"reference/kube-ovn-api/#sgrule","title":"SgRule","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 ipVersion String IP \u7248\u672c\u53f7\uff0c\u53d6\u503c\u4e3a <code>ipv4</code> \u6216\u8005 <code>ipv6</code> protocol String \u53d6\u503c\u4e3a <code>all</code>\u3001<code>icmp</code>\u3001<code>tcp</code> \u6216\u8005 <code>udp</code> priority Int Acl \u4f18\u5148\u7ea7\uff0c\u53d6\u503c\u8303\u56f4\u4e3a 1-200\uff0c\u6570\u503c\u8d8a\u5c0f\uff0c\u4f18\u5148\u7ea7\u8d8a\u9ad8 remoteType String \u53d6\u503c\u4e3a <code>address</code> \u6216\u8005 <code>securityGroup</code> remoteAddress String \u5bf9\u7aef\u5730\u5740 remoteSecurityGroup String \u5bf9\u7aef\u5b89\u5168\u7ec4 portRangeMin Int \u7aef\u53e3\u8303\u56f4\u8d77\u59cb\u503c\uff0c\u6700\u5c0f\u53d6\u503c\u4e3a 1 portRangeMax Int \u7aef\u53e3\u8303\u56f4\u6700\u5927\u503c\uff0c\u6700\u5927\u53d6\u503c\u4e3a 65535 policy String \u53d6\u503c\u4e3a <code>allow</code> \u6216\u8005 <code>drop</code>"},{"location":"reference/kube-ovn-api/#securitygroupstatus","title":"SecurityGroupStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 portGroup String \u5b89\u5168\u7ec4\u5bf9\u5e94\u7684 port-group \u540d\u79f0 allowSameGroupTraffic Bool \u540c\u4e00\u5b89\u5168\u7ec4\u5185\u7684 lsp \u662f\u5426\u53ef\u4ee5\u4e92\u901a\uff0c\u4ee5\u53ca\u5b89\u5168\u7ec4\u7684\u6d41\u91cf\u89c4\u5219\u662f\u5426\u9700\u8981\u66f4\u65b0 ingressMd5 String \u5165\u65b9\u5411\u5b89\u5168\u7ec4\u89c4\u5219 MD5 \u53d6\u503c egressMd5 String \u51fa\u65b9\u5411\u5b89\u5168\u7ec4\u89c4\u5219 MD5 \u53d6\u503c ingressLastSyncSuccess Bool \u5165\u65b9\u5411\u89c4\u5219\u4e0a\u4e00\u6b21\u540c\u6b65\u662f\u5426\u6210\u529f egressLastSyncSuccess Bool \u51fa\u65b9\u5411\u89c4\u5219\u4e0a\u4e00\u6b21\u540c\u6b65\u662f\u5426\u6210\u529f"},{"location":"reference/kube-ovn-api/#vip_1","title":"Vip","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>Vip</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec VipSpec Vip \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status VipStatus Vip \u72b6\u6001\u4fe1\u606f"},{"location":"reference/kube-ovn-api/#vipspec","title":"VipSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 namespace String Vip \u6240\u5728\u547d\u540d\u7a7a\u95f4 subnet String Vip \u6240\u5c5e\u5b50\u7f51 v4ip String Vip v4 IP \u5730\u5740 v6ip String Vip v6 IP \u5730\u5740 macAddress String Vip mac \u5730\u5740 parentV4ip String \u76ee\u524d\u6ca1\u6709\u4f7f\u7528 parentV6ip String \u76ee\u524d\u6ca1\u6709\u4f7f\u7528 parentMac String \u76ee\u524d\u6ca1\u6709\u4f7f\u7528 attachSubnets []String \u8be5\u5b57\u6bb5\u5e9f\u5f03\uff0c\u4e0d\u518d\u4f7f\u7528"},{"location":"reference/kube-ovn-api/#vipstatus","title":"VipStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 conditions []VipCondition Vip \u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49 ready Bool Vip \u662f\u5426\u51c6\u5907\u597d v4ip String Vip v4 IP \u5730\u5740\uff0c\u5e94\u8be5\u548c spec \u5b57\u6bb5\u53d6\u503c\u4e00\u81f4 v6ip String Vip v6 IP \u5730\u5740\uff0c\u5e94\u8be5\u548c spec \u5b57\u6bb5\u53d6\u503c\u4e00\u81f4 mac String Vip mac \u5730\u5740\uff0c\u5e94\u8be5\u548c spec \u5b57\u6bb5\u53d6\u503c\u4e00\u81f4 pv4ip String \u76ee\u524d\u6ca1\u6709\u4f7f\u7528 pv6ip String \u76ee\u524d\u6ca1\u6709\u4f7f\u7528 pmac String \u76ee\u524d\u6ca1\u6709\u4f7f\u7528"},{"location":"reference/kube-ovn-api/#ovneip","title":"OvnEip","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>OvnEip</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec OvnEipSpec \u9ed8\u8ba4 Vpc \u4f7f\u7528 OvnEip \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status OvnEipStatus \u9ed8\u8ba4 Vpc \u4f7f\u7528 OvnEip \u72b6\u6001\u4fe1\u606f"},{"location":"reference/kube-ovn-api/#ovneipspec","title":"OvnEipSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 externalSubnet String OvnEip \u6240\u5728\u7684\u5b50\u7f51\u540d\u79f0 v4ip String OvnEip IP \u5730\u5740 macAddress String OvnEip Mac \u5730\u5740 type String OvnEip \u4f7f\u7528\u7c7b\u578b\uff0c\u53d6\u503c\u6709 <code>fip</code>\u3001<code>snat</code> \u6216\u8005 <code>lrp</code>"},{"location":"reference/kube-ovn-api/#ovneipstatus","title":"OvnEipStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 conditions []OvnEipCondition \u9ed8\u8ba4 Vpc OvnEip \u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49 v4ip String OvnEip \u4f7f\u7528\u7684 v4 IP \u5730\u5740 macAddress String OvnEip \u4f7f\u7528\u7684 Mac \u5730\u5740"},{"location":"reference/kube-ovn-api/#ovnfip","title":"OvnFip","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>OvnFip</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec OvnFipSpec \u9ed8\u8ba4 Vpc \u4f7f\u7528 OvnFip \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status OvnFipStatus \u9ed8\u8ba4 Vpc \u4f7f\u7528 OvnFip \u72b6\u6001\u4fe1\u606f"},{"location":"reference/kube-ovn-api/#ovnfipspec","title":"OvnFipSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 ovnEip String OvnFip \u7ed1\u5b9a\u7684 OvnEip \u540d\u79f0 ipName String OvnFip \u7ed1\u5b9a Pod \u5bf9\u5e94\u7684 IP crd \u540d\u79f0"},{"location":"reference/kube-ovn-api/#ovnfipstatus","title":"OvnFipStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 ready Bool OvnFip \u662f\u5426\u914d\u7f6e\u5b8c\u6210 v4Eip String OvnFip \u7ed1\u5b9a\u7684 OvnEip \u540d\u79f0 v4Ip String OvnFip \u5f53\u524d\u4f7f\u7528\u7684 OvnEip \u5730\u5740 macAddress String OvnFip \u914d\u7f6e\u7684 Mac \u5730\u5740 vpc String OvnFip \u6240\u5728\u7684 Vpc \u540d\u79f0 conditions []OvnFipCondition OvnFip \u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49"},{"location":"reference/kube-ovn-api/#ovnsnatrule","title":"OvnSnatRule","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 apiVersion String \u6807\u51c6 Kubernetes \u7248\u672c\u4fe1\u606f\u5b57\u6bb5\uff0c\u6240\u6709\u81ea\u5b9a\u4e49\u8d44\u6e90\u8be5\u503c\u5747\u4e3a kubeovn.io/v1 kind String \u6807\u51c6 Kubernetes \u8d44\u6e90\u7c7b\u578b\u5b57\u6bb5\uff0c\u672c\u8d44\u6e90\u6240\u6709\u5b9e\u4f8b\u8be5\u503c\u5747\u4e3a <code>OvnSnatRule</code> metadata ObjectMeta \u6807\u51c6 Kubernetes \u8d44\u6e90\u5143\u6570\u636e\u4fe1\u606f spec OvnSnatRuleSpec \u9ed8\u8ba4 Vpc OvnSnatRule \u5177\u4f53\u914d\u7f6e\u4fe1\u606f\u5b57\u6bb5 status OvnSnatRuleStatus \u9ed8\u8ba4 Vpc OvnSnatRule \u72b6\u6001\u4fe1\u606f"},{"location":"reference/kube-ovn-api/#ovnsnatrulespec","title":"OvnSnatRuleSpec","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 ovnEip String OvnSnatRule \u7ed1\u5b9a\u7684 OvnEip \u540d\u79f0 vpcSubnet String OvnSnatRule \u914d\u7f6e\u7684\u5b50\u7f51\u540d\u79f0 ipName String OvnSnatRule \u7ed1\u5b9a Pod \u5bf9\u5e94\u7684 IP crd \u540d\u79f0"},{"location":"reference/kube-ovn-api/#ovnsnatrulestatus","title":"OvnSnatRuleStatus","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 ready Bool OvnSnatRule \u662f\u5426\u914d\u7f6e\u5b8c\u6210 v4Eip String OvnSnatRule \u7ed1\u5b9a\u7684 OvnEip \u5730\u5740 v4IpCidr String \u5728 logical-router \u4e2d\u914d\u7f6e snat \u8f6c\u6362\u4f7f\u7528\u7684 cidr \u5730\u5740 vpc String OvnSnatRule \u6240\u5728\u7684 Vpc \u540d\u79f0 conditions []OvnSnatRuleCondition OvnSnatRule \u72b6\u6001\u53d8\u5316\u4fe1\u606f\uff0c\u5177\u4f53\u5b57\u6bb5\u53c2\u8003\u6587\u6863\u5f00\u5934 Condition \u5b9a\u4e49 <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"reference/kube-ovn-pinger-args/","title":"Kube-OVN-Pinger \u53c2\u6570\u53c2\u8003","text":"<p>\u57fa\u4e8e Kube-OVN v1.12.0 \u7248\u672c\uff0c\u6574\u7406\u4e86 Kube-ovn-pinger \u652f\u6301\u7684\u53c2\u6570\uff0c\u5217\u51fa\u53c2\u6570\u5b9a\u4e49\u5404\u5b57\u6bb5\u7684\u53d6\u503c\u7c7b\u578b\uff0c\u542b\u4e49\u548c\u9ed8\u8ba4\u503c\uff0c\u4ee5\u4f9b\u53c2\u8003</p>"},{"location":"reference/kube-ovn-pinger-args/#_1","title":"\u53c2\u6570\u63cf\u8ff0","text":"\u5c5e\u6027\u540d\u79f0 \u7c7b\u578b \u63cf\u8ff0 \u9ed8\u8ba4\u503c port Int metrics \u7aef\u53e3 8080 kubeconfig String \u5177\u6709\u8ba4\u8bc1\u4fe1\u606f\u7684 kubeconfig \u6587\u4ef6\u8def\u5f84\uff0c \u5982\u679c\u672a\u8bbe\u7f6e\uff0c\u4f7f\u7528 inCluster \u4ee4\u724c\u3002 \"\" ds-namespace String kube-ovn-pinger \u5b88\u62a4\u8fdb\u7a0b\u547d\u540d\u7a7a\u95f4 \"kube-system\" ds-name String kube-ovn-pinger \u5b88\u62a4\u8fdb\u7a0b\u540d\u5b57 \"kube-ovn-pinger\" interval Int \u8fde\u7eed ping \u4e4b\u95f4\u7684\u95f4\u9694\u79d2\u6570 5 mode String \u670d\u52a1\u5668\u6216\u5de5\u4f5c\u6a21\u5f0f \"server\" exit-code Int \u5931\u8d25\u65f6\u9000\u51fa\u4ee3\u7801 0 internal-dns String \u4ece pod \u5185\u89e3\u6790\u5185\u90e8 dns \"kubernetes.default\" external-dns String \u4ece pod \u5185\u89e3\u6790\u5916\u90e8 dns \"\" external-address String \u68c0\u67e5\u4e0e\u5916\u90e8\u5730\u5740\u7684 ping \u8fde\u901a \"114.114.114.114\" network-mode String \u5f53\u524d\u96c6\u7fa4\u4f7f\u7528\u7684 cni \u63d2\u4ef6 \"kube-ovn\" enable-metrics Bool \u662f\u5426\u652f\u6301 metrics \u67e5\u8be2 true ovs.timeout Int \u5bf9 OVS \u7684 JSON-RPC \u8bf7\u6c42\u8d85\u65f6\u3002 2 system.run.dir String OVS \u9ed8\u8ba4\u8fd0\u884c\u76ee\u5f55\u3002 \"/var/run/openvswitch\" database.vswitch.name String OVS \u6570\u636e\u5e93\u7684\u540d\u79f0\u3002 \"Open_vSwitch\" database.vswitch.socket.remote String JSON-RPC unix \u5957\u63a5\u5b57\u5230 OVS \u6570\u636e\u5e93\u3002 \"unix:/var/run/openvswitch/db.sock\" database.vswitch.file.data.path String OVS \u6570\u636e\u5e93\u6587\u4ef6\u3002 \"/etc/openvswitch/conf.db\" database.vswitch.file.log.path String OVS \u6570\u636e\u5e93\u65e5\u5fd7\u6587\u4ef6\u3002 \"/var/log/openvswitch/ovsdb-server.log\" database.vswitch.file.pid.path String OVS \u6570\u636e\u5e93\u8fdb\u7a0b ID \u6587\u4ef6\u3002 \"/var/run/openvswitch/ovsdb-server.pid\" database.vswitch.file.system.id.path String OVS \u7cfb\u7edf\u6807\u8bc6\u6587\u4ef6\u3002 \"/etc/openvswitch/system-id.conf\" service.vswitchd.file.log.path String OVS vswitchd \u5b88\u62a4\u8fdb\u7a0b\u65e5\u5fd7\u6587\u4ef6\u3002 \"/var/log/openvswitch/ovs-vswitchd.log\" service.vswitchd.file.pid.path String OVS vswitchd \u5b88\u62a4\u8fdb\u7a0b\u8fdb\u7a0b ID \u6587\u4ef6\u3002 \"/var/run/openvswitch/ovs-vswitchd.pid\" service.ovncontroller.file.log.path String OVN \u63a7\u5236\u5668\u5b88\u62a4\u8fdb\u7a0b\u65e5\u5fd7\u6587\u4ef6\u3002 \"/var/log/ovn/ovn-controller.log\" service.ovncontroller.file.pid.path String OVN \u63a7\u5236\u5668\u5b88\u62a4\u8fdb\u7a0b\u8fdb\u7a0b ID \u6587\u4ef6\u3002 \"/var/run/ovn/ovn-controller.pid\" <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"reference/metrics/","title":"Kube-OVN \u76d1\u63a7\u6307\u6807","text":"<p>\u672c\u6587\u6863\u5217\u4e3e Kube-OVN \u6240\u63d0\u4f9b\u7684\u76d1\u63a7\u6307\u6807\u3002</p>"},{"location":"reference/metrics/#ovn-monitor","title":"ovn-monitor","text":"<p>OVN \u81ea\u8eab\u72b6\u6001\u76d1\u63a7\u6307\u6807\uff1a</p> \u7c7b\u578b \u6307\u6807\u9879 \u63cf\u8ff0 Gauge kube_ovn_ovn_status OVN \u89d2\u8272\u72b6\u6001\uff0c (2) \u4e3a follower\uff1b (1) \u4e3a leader, (0) \u4e3a\u5f02\u5e38\u72b6\u6001\u3002 Gauge kube_ovn_failed_req_count OVN \u5931\u8d25\u8bf7\u6c42\u6570\u91cf\u3002 Gauge kube_ovn_log_file_size_bytes OVN \u7ec4\u4ef6\u65e5\u5fd7\u6587\u4ef6\u5927\u5c0f\u3002 Gauge kube_ovn_db_file_size_bytes OVN \u7ec4\u4ef6\u6570\u636e\u5e93\u6587\u4ef6\u5927\u5c0f\u3002 Gauge kube_ovn_chassis_info OVN chassis \u72b6\u6001 (1) \u8fd0\u884c\u4e2d\uff0c(0) \u505c\u6b62\u3002 Gauge kube_ovn_db_status OVN \u6570\u636e\u5e93\u72b6\u6001, (1) \u4e3a\u6b63\u5e38\uff1b (0) \u4e3a\u5f02\u5e38\u3002 Gauge kube_ovn_logical_switch_info OVN logical switch \u4fe1\u606f\uff0c\u503c\u4e3a (1)\uff0c\u6807\u7b7e\u4e2d\u5305\u542b logical switch \u540d\u5b57\u3002 Gauge kube_ovn_logical_switch_external_id OVN logical switch external_id \u4fe1\u606f\uff0c\u503c\u4e3a (1)\uff0c\u6807\u7b7e\u4e2d\u5305\u542b external-id \u5185\u5bb9\u3002 Gauge kube_ovn_logical_switch_port_binding OVN logical switch \u548c logical switch port \u5173\u8054\u4fe1\u606f\uff0c\u503c\u4e3a (1)\uff0c\u901a\u8fc7\u6807\u7b7e\u8fdb\u884c\u5173\u8054\u3002 Gauge kube_ovn_logical_switch_tunnel_key \u548c OVN logical switch \u5173\u8054\u7684 tunnel key \u4fe1\u606f\u3002 Gauge kube_ovn_logical_switch_ports_num OVN logical switch \u4e0a logical port \u7684\u6570\u91cf\u3002 Gauge kube_ovn_logical_switch_port_info OVN logical switch port \u4fe1\u606f\uff0c\u503c\u4e3a (1)\uff0c\u6807\u7b7e\u4e2d\u5305\u542b\u5177\u4f53\u4fe1\u606f\u3002 Gauge kube_ovn_logical_switch_port_tunnel_key \u548c OVN logical switch port \u5173\u8054\u7684 tunnel key \u4fe1\u606f\u3002 Gauge kube_ovn_cluster_enabled (1) OVN \u6570\u636e\u5e93\u4e3a\u96c6\u7fa4\u6a21\u5f0f\uff1b (0) OVN \u6570\u636e\u5e93\u4e3a\u975e\u96c6\u7fa4\u6a21\u5f0f\u3002 Gauge kube_ovn_cluster_role \u6bcf\u4e2a\u6570\u636e\u5e93\u5b9e\u4f8b\u7684\u89d2\u8272\uff0c\u503c\u4e3a (1)\uff0c\u6807\u7b7e\u4e2d\u5305\u542b\u5bf9\u5e94\u89d2\u8272\u4fe1\u606f\u3002 Gauge kube_ovn_cluster_status \u6bcf\u4e2a\u6570\u636e\u5e93\u5b9e\u4f8b\u7684\u72b6\u6001\uff0c\u503c\u4e3a (1)\uff0c\u6807\u7b7e\u4e2d\u5305\u542b\u5bf9\u5e94\u72b6\u6001\u4fe1\u606f\u3002 Gauge kube_ovn_cluster_term RAFT term \u4fe1\u606f\u3002 Gauge kube_ovn_cluster_leader_self \u5f53\u524d\u6570\u636e\u5e93\u5b9e\u4f8b\u662f\u5426\u4e3a leader (1) \u662f\uff0c (0) \u4e0d\u662f\u3002 Gauge kube_ovn_cluster_vote_self \u5f53\u524d\u6570\u636e\u5e93\u5b9e\u4f8b\u662f\u5426\u9009\u4e3e\u81ea\u5df1\u4e3a leader (1) \u662f\uff0c (0) \u4e0d\u662f\u3002 Gauge kube_ovn_cluster_election_timer \u5f53\u524d election timer \u503c\u3002 Gauge kube_ovn_cluster_log_not_committed \u672a commit \u7684 RAFT \u65e5\u5fd7\u6570\u91cf\u3002 Gauge kube_ovn_cluster_log_not_applied \u672a apply \u7684 RAFT \u65e5\u5fd7\u6570\u91cf\u3002 Gauge kube_ovn_cluster_log_index_start \u5f53\u524d RAFT \u65e5\u5fd7\u6761\u76ee\u7684\u8d77\u59cb\u503c\u3002 Gauge kube_ovn_cluster_log_index_next RAFT \u65e5\u5fd7\u6761\u76ee\u7684\u4e0b\u4e00\u4e2a\u503c\u3002 Gauge kube_ovn_cluster_inbound_connections_total \u5f53\u524d\u5b9e\u4f8b\u7684\u5165\u5411\u8fde\u63a5\u6570\u91cf\u3002 Gauge kube_ovn_cluster_outbound_connections_total \u5f53\u524d\u5b9e\u4f8b\u7684\u51fa\u5411\u8fde\u63a5\u6570\u91cf\u3002 Gauge kube_ovn_cluster_inbound_connections_error_total \u5f53\u524d\u5b9e\u4f8b\u7684\u5165\u5411\u9519\u8bef\u8fde\u63a5\u6570\u91cf\u3002 Gauge kube_ovn_cluster_outbound_connections_error_total \u5f53\u524d\u5b9e\u4f8b\u7684\u51fa\u5411\u9519\u8bef\u8fde\u63a5\u6570\u91cf\u3002"},{"location":"reference/metrics/#ovs-monitor","title":"ovs-monitor","text":"<p><code>ovsdb</code> \u548c <code>vswitchd</code> \u81ea\u8eab\u72b6\u6001\u76d1\u63a7\u6307\u6807\uff1a</p> \u7c7b\u578b \u6307\u6807\u9879 \u63cf\u8ff0 Gauge ovs_status OVS \u5065\u5eb7\u72b6\u6001\uff0c (1) \u4e3a\u6b63\u5e38\uff0c(0) \u4e3a\u5f02\u5e38\u3002 Gauge ovs_info OVS \u57fa\u7840\u4fe1\u606f\uff0c\u503c\u4e3a (1)\uff0c\u6807\u7b7e\u4e2d\u5305\u542b\u5bf9\u5e94\u4fe1\u606f\u3002 Gauge failed_req_count OVS \u5931\u8d25\u8bf7\u6c42\u6570\u91cf\u3002 Gauge log_file_size OVS \u7ec4\u4ef6\u65e5\u5fd7\u6587\u4ef6\u5927\u5c0f\u3002 Gauge db_file_size OVS \u7ec4\u4ef6\u6570\u636e\u5e93\u6587\u4ef6\u5927\u5c0f\u3002 Gauge datapath Datapath \u57fa\u7840\u4fe1\u606f\uff0c\u503c\u4e3a (1)\uff0c\u6807\u7b7e\u4e2d\u5305\u542b\u5bf9\u5e94\u4fe1\u606f\u3002 Gauge dp_total \u5f53\u524d OVS \u4e2d datapath \u6570\u91cf\u3002 Gauge dp_if Datapath \u63a5\u53e3\u57fa\u7840\u4fe1\u606f\uff0c\u503c\u4e3a (1)\uff0c\u6807\u7b7e\u4e2d\u5305\u542b\u5bf9\u5e94\u4fe1\u606f\u3002 Gauge dp_if_total \u5f53\u524d datapath \u4e2d port \u6570\u91cf\u3002 Gauge dp_flows_total Datapath \u4e2d flow \u6570\u91cf\u3002 Gauge dp_flows_lookup_hit Datapath \u4e2d\u547d\u4e2d\u5f53\u524d flow \u6570\u636e\u5305\u6570\u91cf\u3002 Gauge dp_flows_lookup_missed Datapath \u4e2d\u672a\u547d\u4e2d\u5f53\u524d flow \u6570\u636e\u5305\u6570\u91cf\u3002 Gauge dp_flows_lookup_lost Datapath \u4e2d\u9700\u8981\u53d1\u9001\u7ed9 userspace \u5904\u7406\u7684\u6570\u636e\u5305\u6570\u91cf\u3002 Gauge dp_masks_hit Datapath \u4e2d\u547d\u4e2d\u5f53\u524d mask \u6570\u636e\u5305\u6570\u91cf\u3002 Gauge dp_masks_total Datapath \u4e2d mask \u7684\u6570\u91cf\u3002 Gauge dp_masks_hit_ratio Datapath \u4e2d \u6570\u636e\u5305\u547d\u4e2d mask \u7684\u6bd4\u7387\u3002 Gauge interface OVS \u63a5\u53e3\u57fa\u7840\u4fe1\u606f\uff0c\u503c\u4e3a (1)\uff0c\u6807\u7b7e\u4e2d\u5305\u542b\u5bf9\u5e94\u4fe1\u606f\u3002 Gauge interface_admin_state \u63a5\u53e3\u7ba1\u7406\u72b6\u6001\u4fe1\u606f (0) \u4e3a down, (1) \u4e3a up, (2) \u4e3a\u5176\u4ed6\u72b6\u6001\u3002 Gauge interface_link_state \u63a5\u53e3\u94fe\u8def\u72b6\u6001\u4fe1\u606f (0) \u4e3a down, (1) \u4e3a up, (2) \u4e3a\u5176\u4ed6\u72b6\u6001\u3002 Gauge interface_mac_in_use OVS Interface \u4f7f\u7528\u7684 MAC \u5730\u5740 Gauge interface_mtu OVS Interface \u4f7f\u7528\u7684 MTU\u3002 Gauge interface_of_port OVS Interface \u5173\u8054\u7684 OpenFlow Port ID\u3002 Gauge interface_if_index OVS Interface \u5173\u8054\u7684 Index\u3002 Gauge interface_tx_packets OVS Interface \u53d1\u9001\u5305\u6570\u91cf\u3002 Gauge interface_tx_bytes OVS Interface \u53d1\u9001\u5305\u5927\u5c0f\u3002 Gauge interface_rx_packets OVS Interface \u63a5\u6536\u5305\u6570\u91cf\u3002 Gauge interface_rx_bytes OVS Interface \u63a5\u6536\u5305\u5927\u5c0f\u3002 Gauge interface_rx_crc_err OVS Interface \u63a5\u6536\u5305\u6821\u9a8c\u548c\u9519\u8bef\u6570\u91cf\u3002 Gauge interface_rx_dropped OVS Interface \u63a5\u6536\u5305\u4e22\u5f03\u6570\u91cf\u3002 Gauge interface_rx_errors OVS Interface \u63a5\u6536\u5305\u9519\u8bef\u6570\u91cf\u3002 Gauge interface_rx_frame_err OVS Interface \u63a5\u6536\u5e27\u9519\u8bef\u6570\u91cf\u3002 Gauge interface_rx_missed_err OVS Interface \u63a5\u6536\u5305 miss \u6570\u91cf\u3002 Gauge interface_rx_over_err OVS Interface \u63a5\u6536\u5305 overrun \u6570\u91cf\u3002 Gauge interface_tx_dropped OVS Interface \u53d1\u9001\u5305\u4e22\u5f03\u6570\u91cf\u3002 Gauge interface_tx_errors OVS Interface \u53d1\u9001\u5305\u9519\u8bef\u6570\u91cf\u3002 Gauge interface_collisions OVS interface \u51b2\u7a81\u6570\u91cf\u3002"},{"location":"reference/metrics/#kube-ovn-pinger","title":"kube-ovn-pinger","text":"<p>\u7f51\u7edc\u8d28\u91cf\u76f8\u5173\u76d1\u63a7\u6307\u6807\uff1a</p> \u7c7b\u578b \u6307\u6807\u9879 \u63cf\u8ff0 Gauge pinger_ovs_up \u8282\u70b9 OVS \u8fd0\u884c\u3002 Gauge pinger_ovs_down \u8282\u70b9 OVS \u505c\u6b62\u3002 Gauge pinger_ovn_controller_up \u8282\u70b9 ovn-controller \u8fd0\u884c\u3002 Gauge pinger_ovn_controller_down \u8282\u70b9 ovn-controller \u505c\u6b62\u3002 Gauge pinger_inconsistent_port_binding OVN-SB \u91cc portbinding \u6570\u91cf\u548c\u4e3b\u673a OVS interface \u4e0d\u4e00\u81f4\u7684\u6570\u91cf\u3002 Gauge pinger_apiserver_healthy kube-ovn-pinger \u53ef\u4ee5\u8054\u901a apiserver\u3002 Gauge pinger_apiserver_unhealthy kube-ovn-pinger \u65e0\u6cd5\u8054\u901a apiserver\u3002 Histogram pinger_apiserver_latency_ms kube-ovn-pinger \u8bbf\u95ee apiserver \u5ef6\u8fdf\u3002 Gauge pinger_internal_dns_healthy kube-ovn-pinger \u53ef\u4ee5\u89e3\u6790\u5185\u90e8\u57df\u540d\u3002 Gauge pinger_internal_dns_unhealthy kube-ovn-pinger \u65e0\u6cd5\u89e3\u6790\u5185\u90e8\u57df\u540d\u3002 Histogram pinger_internal_dns_latency_ms kube-ovn-pinger \u89e3\u6790\u5185\u90e8\u57df\u540d\u5ef6\u8fdf\u3002 Gauge pinger_external_dns_health kube-ovn-pinger \u53ef\u4ee5\u89e3\u6790\u5916\u90e8\u57df\u540d\u3002 Gauge pinger_external_dns_unhealthy kube-ovn-pinger \u65e0\u6cd5\u89e3\u6790\u5916\u90e8\u57df\u540d\u3002 Histogram pinger_external_dns_latency_ms kube-ovn-pinger \u89e3\u6790\u5916\u90e8\u57df\u540d\u5ef6\u8fdf\u3002 Histogram pinger_pod_ping_latency_ms kube-ovn-pinger ping Pod \u5ef6\u8fdf\u3002 Gauge pinger_pod_ping_lost_total kube-ovn-pinger ping Pod \u4e22\u5305\u6570\u91cf\u3002 Gauge pinger_pod_ping_count_total kube-ovn-pinger ping Pod \u6570\u91cf\u3002 Histogram pinger_node_ping_latency_ms kube-ovn-pinger ping Node \u5ef6\u8fdf\u3002 Gauge pinger_node_ping_lost_total kube-ovn-pinger ping Node \u4e22\u5305\u3002 Gauge pinger_node_ping_count_total kube-ovn-pinger ping Node \u6570\u91cf\u3002 Histogram pinger_external_ping_latency_ms kube-ovn-pinger ping \u5916\u90e8\u5730\u5740 \u5ef6\u8fdf\u3002 Gauge pinger_external_lost_total kube-ovn-pinger ping \u5916\u90e8\u4e22\u5305\u6570\u91cf\u3002"},{"location":"reference/metrics/#kube-ovn-controller","title":"kube-ovn-controller","text":"<p><code>kube-ovn-controller</code> \u76f8\u5173\u76d1\u63a7\u6307\u6807\uff1a</p> \u7c7b\u578b \u6307\u6807\u9879 \u63cf\u8ff0 Histogram rest_client_request_latency_seconds \u8bf7\u6c42 apiserver \u5ef6\u8fdf\u3002 Counter rest_client_requests_total \u8bf7\u6c42 apiserver \u6570\u91cf\u3002 Counter lists_total API list \u8bf7\u6c42\u6570\u91cf\u3002 Summary list_duration_seconds API list \u8bf7\u6c42\u5ef6\u8fdf\u3002 Summary items_per_list API list \u8fd4\u56de\u7ed3\u679c\u6570\u91cf\u3002 Counter watches_total API watch \u8bf7\u6c42\u6570\u91cf\u3002 Counter short_watches_total \u77ed\u65f6\u95f4 API watch \u8bf7\u6c42\u6570\u91cf\u3002 Summary watch_duration_seconds API watch \u6301\u7eed\u65f6\u95f4\u3002 Summary items_per_watch API watch \u8fd4\u56de\u7ed3\u679c\u6570\u91cf\u3002 Gauge last_resource_version \u6700\u65b0\u7684 resource version\u3002 Histogram ovs_client_request_latency_milliseconds \u8bf7\u6c42 OVN \u7ec4\u4ef6\u5ef6\u8fdf\u3002 Gauge subnet_available_ip_count \u5b50\u7f51\u53ef\u7528 IP \u6570\u91cf\u3002 Gauge subnet_used_ip_count \u5b50\u7f51\u5df2\u7528 IP \u6570\u91cf\u3002"},{"location":"reference/metrics/#kube-ovn-cni","title":"kube-ovn-cni","text":"<p><code>kube-ovn-cni</code> \u76f8\u5173\u76d1\u63a7\u6307\u6807\uff1a</p> \u7c7b\u578b \u6307\u6807\u9879 \u63cf\u8ff0 Histogram cni_op_latency_seconds CNI \u64cd\u4f5c\u5ef6\u8fdf\u3002 Counter cni_wait_address_seconds_total CNI \u7b49\u5f85\u5730\u5740\u5c31\u7eea\u65f6\u95f4\u3002 Counter cni_wait_connectivity_seconds_total CNI \u7b49\u5f85\u8fde\u63a5\u5c31\u7eea\u65f6\u95f4\u3002 Counter cni_wait_route_seconds_total CNI \u7b49\u5f85\u8def\u7531\u5c31\u7eea\u65f6\u95f4\u3002 Histogram rest_client_request_latency_seconds \u8bf7\u6c42 apiserver \u5ef6\u8fdf\u3002 Counter rest_client_requests_total \u8bf7\u6c42 apiserver \u6570\u91cf\u3002 Counter lists_total API list \u8bf7\u6c42\u6570\u91cf\u3002 Summary list_duration_seconds API list \u8bf7\u6c42\u5ef6\u8fdf\u3002 Summary items_per_list API list \u8fd4\u56de\u7ed3\u679c\u6570\u91cf\u3002 Counter watches_total API watch \u8bf7\u6c42\u6570\u91cf\u3002 Counter short_watches_total \u77ed\u65f6\u95f4 API watch \u8bf7\u6c42\u6570\u91cf\u3002 Summary watch_duration_seconds API watch \u6301\u7eed\u65f6\u95f4\u3002 Summary items_per_watch API watch \u8fd4\u56de\u7ed3\u679c\u6570\u91cf\u3002 Gauge last_resource_version \u6700\u65b0\u7684 resource version\u3002 Histogram ovs_client_request_latency_milliseconds \u8bf7\u6c42 OVN \u7ec4\u4ef6\u5ef6\u8fdf\u3002 <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"reference/ovs-ovn-customized/","title":"\u5bf9\u4e0a\u6e38 OVS/OVN \u4fee\u6539","text":"<p>\u4e0a\u6e38 OVN/OVS \u6700\u521d\u8bbe\u8ba1\u76ee\u6807\u4e3a\u901a\u7528 SDN \u63a7\u5236\u5668\u548c\u6570\u636e\u5e73\u9762\u3002\u7531\u4e8e Kubernetes \u7f51\u7edc\u5b58\u5728\u4e00\u4e9b\u7279\u6b8a\u7684\u7528\u6cd5\uff0c \u5e76\u4e14 Kube-OVN \u53ea\u91cd\u70b9\u4f7f\u7528\u4e86\u90e8\u5206\u529f\u80fd\uff0c\u4e3a\u4e86 \u8fbe\u5230\u66f4\u597d\u7684\u6027\u80fd\u3001\u7a33\u5b9a\u6027\u548c\u7279\u5b9a\u7684\u529f\u80fd\uff0cKube-OVN \u5bf9\u4e0a\u6e38 OVN/OVS \u505a\u4e86\u90e8\u5206\u4fee\u6539\u3002\u7528\u6237\u5982\u679c\u4f7f\u7528\u81ea\u5df1\u7684 OVN/OVS \u914d\u5408 Kube-OVN \u7684\u63a7\u5236\u5668\u8fdb\u884c\u5de5\u4f5c\u65f6\u9700\u8981\u6ce8\u610f \u4e0b\u8ff0\u7684\u6539\u52a8\u53ef\u80fd\u9020\u6210\u7684\u5f71\u54cd\u3002</p> <p>\u672a\u5408\u5165\u4e0a\u6e38\u4fee\u6539\uff1a</p> <ul> <li>38df6fa3f7 \u8c03\u6574\u9009\u4e3e timer\uff0c\u907f\u514d\u5927\u89c4\u6a21\u96c6\u7fa4\u9009\u4e3e\u6296\u52a8\u3002</li> <li>d4888c4e75 \u6dfb\u52a0 fdb \u66f4\u65b0\u65e5\u5fd7\u3002</li> <li>d4888c4e75 \u4fee\u590d hairpin \u73af\u5883\u4e0b fdb \u5b66\u4e60\u9519\u8bef\u7684\u95ee\u9898\u3002</li> <li>9a81b91368 \u4e3a ovsdb-tool \u7684 join-cluster \u5b50\u547d\u4ee4\u6dfb\u52a0 Server ID \u53c2\u6570\u3002</li> <li>62d4969877 \u4fee\u590d\u5f00\u542f SSL \u540e OVSDB \u76d1\u542c\u5730\u5740\u9519\u8bef\u7684\u95ee\u9898\u3002</li> <li>0700cb90f9 \u76ee\u7684\u5730\u5740\u975e Service \u6d41\u91cf\u7ed5\u8fc7 conntrack \u4ee5\u63d0\u9ad8\u7279\u5b9a\u6570\u636e\u94fe\u8def\u6027\u80fd\u3002</li> <li>c48049a64f ECMP \u7b97\u6cd5\u7531 dp_hash \u8c03\u6574\u4e3a hash\uff0c\u907f\u514d\u90e8\u5206\u5185\u6838\u51fa\u73b0\u7684\u54c8\u5e0c\u9519\u8bef\u95ee\u9898\u3002</li> <li>64383c14a9 \u4fee\u590d Windows \u4e0b\u5185\u6838 Crash \u95ee\u9898\u3002</li> <li>08a95db2ca \u652f\u6301 Windows \u4e0b\u7684 github action \u6784\u5efa\u3002</li> <li>680e77a190 Windows \u4e0b\u9ed8\u8ba4\u4f7f\u7528 tcp \u76d1\u542c\u3002</li> <li>05e57b3227 \u652f\u6301 Windows \u7f16\u8bd1\u3002</li> <li>b3801ecb73 \u4fee\u6539\u6e90\u8def\u7531\u7684\u4f18\u5148\u7ea7\u3002</li> <li>977e569539 \u4fee\u590d Underlay \u6a21\u5f0f\u4e0b Pod \u6570\u91cf\u8fc7\u591a\u5bfc\u81f4 OVS \u6d41\u8868 resubmit \u6b21\u6570\u8d85\u8fc7\u4e0a\u9650\u7684\u95ee\u9898\u3002</li> <li>45a4a22161 ovn-nbctl\uff1avips \u4e3a\u7a7a\u65f6\u4e0d\u5220\u9664 Load Balancer\u3002</li> <li>540592b9ff DNAT \u540e\u66ff\u6362 Mac \u5730\u5740\u4e3a\u76ee\u6807\u5730\u5740\uff0c\u51cf\u5c11\u989d\u5916\u6027\u80fd\u5f00\u9500\u3002</li> <li>10972d9632 \u4fee\u590d vswitchd ofport_usage \u5185\u5b58\u6cc4\u9732\u3002</li> </ul> <p>\u5df2\u5408\u5165\u4e0a\u6e38\u4fee\u6539\uff1a</p> <ul> <li>20626ea909 \u7ec4\u64ad\u6d41\u91cf\u7ed5\u8fc7 LB \u548c ACL \u5904\u7406\u9636\u6bb5\uff0c\u4ee5\u63d0\u9ad8\u7279\u5b9a\u6570\u636e\u94fe\u8def\u6027\u80fd\u3002</li> <li>a2d9ff3ccd Deb \u6784\u5efa\u589e\u52a0\u7f16\u8bd1\u4f18\u5316\u9009\u9879\u3002</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"reference/tunnel-protocol/","title":"\u96a7\u9053\u534f\u8bae\u8bf4\u660e","text":"<p>Kube-OVN \u4f7f\u7528 OVN/OVS \u4f5c\u4e3a\u6570\u636e\u5e73\u9762\u5b9e\u73b0\uff0c\u76ee\u524d\u652f\u6301 <code>Geneve</code>\uff0c<code>Vxlan</code> \u548c <code>STT</code> \u4e09\u79cd\u96a7\u9053\u5c01\u88c5\u534f\u8bae\u3002 \u8fd9\u4e09\u79cd\u534f\u8bae\u5728\u529f\u80fd\uff0c\u6027\u80fd\u548c\u6613\u7528\u6027\u4e0a\u5b58\u5728\u7740\u533a\u522b\uff0c\u672c\u6587\u6863\u5c06\u4ecb\u7ecd\u4e09\u79cd\u534f\u8bae\u5728\u4f7f\u7528\u4e2d\u7684\u5dee\u5f02\uff0c\u7528\u6237\u53ef\u6839\u636e\u81ea\u5df1\u7684\u60c5\u51b5\u8fdb\u884c\u9009\u62e9\u3002</p>"},{"location":"reference/tunnel-protocol/#geneve","title":"Geneve","text":"<p><code>Geneve</code> \u534f\u8bae\u4e3a Kube-OVN \u90e8\u7f72\u65f6\u9009\u62e9\u7684\u9ed8\u8ba4\u96a7\u9053\u534f\u8bae\uff0c\u4e5f\u662f OVN \u9ed8\u8ba4\u63a8\u8350\u7684\u96a7\u9053\u534f\u8bae\u3002\u8be5\u534f\u8bae\u5728\u5185\u6838\u4e2d\u5f97\u5230\u4e86\u5e7f\u6cdb\u7684\u652f\u6301\uff0c \u5e76\u53ef\u4ee5\u5229\u7528\u73b0\u4ee3\u7f51\u5361\u7684\u901a\u7528 Offload \u80fd\u529b\u8fdb\u884c\u52a0\u901f\u3002\u7531\u4e8e <code>Geneve</code> \u6709\u7740\u53ef\u53d8\u957f\u7684\u5934\u90e8\uff0c\u53ef\u4ee5\u4f7f\u7528 24bit \u7a7a\u95f4\u6765\u6807\u5fd7\u4e0d\u540c\u7684 datapath \u7528\u6237\u53ef\u4ee5\u521b\u5efa\u66f4\u591a\u6570\u91cf\u7684\u865a\u62df\u7f51\u7edc\u3002</p> <p>\u5982\u679c\u4f7f\u7528 Mellanox \u6216\u82af\u542f\u6e90\u7684\u667a\u80fd\u7f51\u5361 OVS \u5378\u8f7d\uff0c<code>Geneve</code> \u9700\u8981\u8f83\u9ad8\u7248\u672c\u7684\u5185\u6838\u652f\u6301\uff0c\u9700\u8981\u9009\u62e9 5.4 \u4ee5\u4e0a\u7684\u4e0a\u6e38\u5185\u6838\uff0c \u6216 backport \u4e86\u8be5\u529f\u80fd\u7684\u5176\u4ed6\u517c\u5bb9\u5185\u6838\u3002</p> <p>\u7531\u4e8e\u4f7f\u7528 UDP \u8fdb\u884c\u5c01\u88c5\uff0c\u8be5\u534f\u8bae\u5728\u5904\u7406 TCP over UDP \u65f6\u4e0d\u80fd\u5f88\u597d\u7684\u5229\u7528\u73b0\u4ee3\u7f51\u5361\u7684 TCP \u76f8\u5173\u5378\u8f7d\uff0c\u5728\u5904\u7406\u5927\u5305\u65f6\u4f1a\u6d88\u8017\u8f83\u591a CPU \u8d44\u6e90\u3002</p>"},{"location":"reference/tunnel-protocol/#vxlan","title":"Vxlan","text":"<p><code>Vxlan</code> \u4e3a\u4e0a\u6e38 OVN \u8fd1\u671f\u652f\u6301\u7684\u534f\u8bae\uff0c\u8be5\u534f\u8bae\u5728\u5185\u6838\u4e2d\u5f97\u5230\u4e86\u5e7f\u6cdb\u7684\u652f\u6301\uff0c \u5e76\u53ef\u4ee5\u5229\u7528\u73b0\u4ee3\u7f51\u5361\u7684\u901a\u7528 Offload \u80fd\u529b\u8fdb\u884c\u52a0\u901f\u3002 \u7531\u4e8e\u8be5\u534f\u8bae\u5934\u90e8\u957f\u5ea6\u6709\u9650\uff0c\u5e76\u4e14 OVN \u9700\u8981\u4f7f\u7528\u989d\u5916\u7684\u7a7a\u95f4\u8fdb\u884c\u7f16\u6392\uff0cdatapath \u7684\u6570\u91cf\u5b58\u5728\u9650\u5236\uff0c\u6700\u591a\u53ea\u80fd\u521b\u5efa 4096 \u4e2a datapath\uff0c \u6bcf\u4e2a datapath \u4e0b\u6700\u591a 4096 \u4e2a\u7aef\u53e3\u3002\u540c\u65f6\u7531\u4e8e\u7a7a\u95f4\u6709\u9650\uff0c\u57fa\u4e8e <code>inport</code> \u7684 ACL \u6ca1\u6709\u8fdb\u884c\u652f\u6301\u3002</p> <p>\u5982\u679c\u4f7f\u7528 Mellanox \u6216\u82af\u542f\u6e90\u7684\u667a\u80fd\u7f51\u5361 OVS \u5378\u8f7d\uff0c<code>Vxlan</code> \u7684\u5378\u8f7d\u5728\u5e38\u89c1\u5185\u6838\u4e2d\u5df2\u83b7\u5f97\u652f\u6301\u3002</p> <p>\u7531\u4e8e\u4f7f\u7528 UDP \u8fdb\u884c\u5c01\u88c5\uff0c\u8be5\u534f\u8bae\u5728\u5904\u7406 TCP over UDP \u65f6\u4e0d\u80fd\u5f88\u597d\u7684\u5229\u7528\u73b0\u4ee3\u7f51\u5361\u7684 TCP \u76f8\u5173\u5378\u8f7d\uff0c\u5728\u5904\u7406\u5927\u5305\u65f6\u4f1a\u6d88\u8017\u8f83\u591a CPU \u8d44\u6e90\u3002</p>"},{"location":"reference/tunnel-protocol/#stt","title":"STT","text":"<p><code>STT</code> \u534f\u8bae\u4e3a OVN \u8f83\u65e9\u652f\u6301\u7684\u96a7\u9053\u534f\u8bae\uff0c\u8be5\u534f\u8bae\u4f7f\u7528\u7c7b TCP \u7684\u5934\u90e8\uff0c\u53ef\u4ee5\u5145\u5206\u5229\u7528\u73b0\u4ee3\u7f51\u5361\u901a\u7528\u7684 TCP \u5378\u8f7d\u80fd\u529b\uff0c\u5927\u5e45\u63d0\u5347 TCP \u7684\u541e\u5410\u91cf\u3002\u540c\u65f6\u8be5\u534f\u8bae\u5934\u90e8\u8f83\u957f\u53ef\u652f\u6301\u5b8c\u6574\u7684 OVN \u80fd\u529b\u548c\u5927\u89c4\u6a21\u7684 datapath\u3002</p> <p>\u8be5\u534f\u8bae\u672a\u5728\u5185\u6838\u4e2d\u652f\u6301\uff0c\u82e5\u8981\u4f7f\u7528\u9700\u8981\u989d\u5916\u7f16\u8bd1 OVS \u5185\u6838\u6a21\u5757\uff0c\u5e76\u5728\u5347\u7ea7\u5185\u6838\u65f6\u5bf9\u5e94\u518d\u6b21\u7f16\u8bd1\u65b0\u7248\u672c\u5185\u6838\u6a21\u5757\u3002</p> <p>\u8be5\u534f\u8bae\u76ee\u524d\u672a\u88ab\u667a\u80fd\u7f51\u5361\u652f\u6301\uff0c\u65e0\u6cd5\u4f7f\u7528 OVS \u7684\u5378\u8f7d\u80fd\u529b\u3002</p>"},{"location":"reference/tunnel-protocol/#_2","title":"\u53c2\u8003\u8d44\u6599","text":"<ul> <li>https://ipwithease.com/vxlan-vs-geneve-understand-the-difference/</li> <li>OVN FAQ</li> <li>What is Geneve</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"reference/underlay-topology/","title":"Underlay \u6d41\u91cf\u62d3\u6251","text":"<p>\u672c\u6587\u6863\u4ecb\u7ecd Underlay \u6a21\u5f0f\u4e0b\u6d41\u91cf\u5728\u4e0d\u540c\u60c5\u51b5\u4e0b\u7684\u8f6c\u53d1\u8def\u5f84\u3002</p>"},{"location":"reference/underlay-topology/#_1","title":"\u540c\u8282\u70b9\u540c\u5b50\u7f51","text":"<p>\u5185\u90e8\u903b\u8f91\u4ea4\u6362\u673a\u76f4\u63a5\u4ea4\u6362\u6570\u636e\u5305\uff0c\u4e0d\u8fdb\u5165\u5916\u90e8\u7f51\u7edc\u3002</p> <p></p>"},{"location":"reference/underlay-topology/#_2","title":"\u8de8\u8282\u70b9\u540c\u5b50\u7f51","text":"<p>\u6570\u636e\u5305\u7ecf\u7531\u8282\u70b9\u7f51\u5361\u8fdb\u5165\u5916\u90e8\u4ea4\u6362\u673a\uff0c\u7531\u5916\u90e8\u4ea4\u6362\u673a\u8fdb\u884c\u4ea4\u6362\u3002</p> <p></p>"},{"location":"reference/underlay-topology/#_3","title":"\u540c\u8282\u70b9\u4e0d\u540c\u5b50\u7f51","text":"<p>\u6570\u636e\u5305\u7ecf\u7531\u8282\u70b9\u7f51\u5361\u8fdb\u5165\u5916\u90e8\u7f51\u7edc\uff0c\u7531\u5916\u90e8\u4ea4\u6362\u673a\u53ca\u8def\u7531\u5668\u8fdb\u884c\u4ea4\u6362\u548c\u8def\u7531\u8f6c\u53d1\u3002</p> <p></p> <p>\u6b64\u5904 br-provider-1 \u548c br-provider-2 \u53ef\u4ee5\u662f\u540c\u4e00\u4e2a OVS \u7f51\u6865\uff0c\u5373\u591a\u4e2a\u4e0d\u540c\u5b50\u7f51\u53ef\u4ee5\u4f7f\u7528\u540c\u4e00\u4e2a Provider Network\u3002</p>"},{"location":"reference/underlay-topology/#_4","title":"\u8de8\u8282\u70b9\u4e0d\u540c\u5b50\u7f51","text":"<p>\u6570\u636e\u5305\u7ecf\u7531\u8282\u70b9\u7f51\u5361\u8fdb\u5165\u5916\u90e8\u7f51\u7edc\uff0c\u7531\u5916\u90e8\u4ea4\u6362\u673a\u53ca\u8def\u7531\u5668\u8fdb\u884c\u4ea4\u6362\u548c\u8def\u7531\u8f6c\u53d1\u3002</p> <p></p>"},{"location":"reference/underlay-topology/#_5","title":"\u8bbf\u95ee\u5916\u90e8","text":"<p>\u6570\u636e\u5305\u7ecf\u7531\u8282\u70b9\u7f51\u5361\u8fdb\u5165\u5916\u90e8\u7f51\u7edc\uff0c\u7531\u5916\u90e8\u4ea4\u6362\u673a\u53ca\u8def\u7531\u5668\u8fdb\u884c\u4ea4\u6362\u548c\u8def\u7531\u8f6c\u53d1\u3002</p> <p></p> <p>\u8282\u70b9\u4e0e Pod \u4e4b\u95f4\u7684\u901a\u4fe1\u5927\u4f53\u4e0a\u4e5f\u9075\u5faa\u6b64\u903b\u8f91\u3002</p>"},{"location":"reference/underlay-topology/#vlan-tag","title":"\u65e0 Vlan Tag \u4e0b\u603b\u89c8","text":""},{"location":"reference/underlay-topology/#vlan","title":"\u591a VLAN \u603b\u89c8","text":""},{"location":"reference/underlay-topology/#pod-service-ip","title":"Pod \u8bbf\u95ee Service IP","text":"<p>Kube-OVN \u4e3a\u6bcf\u4e2a Kubernetes Service \u5728\u6bcf\u4e2a\u5b50\u7f51\u7684\u903b\u8f91\u4ea4\u6362\u673a\u4e0a\u914d\u7f6e\u4e86\u8d1f\u8f7d\u5747\u8861\u3002 \u5f53 Pod \u901a\u8fc7\u8bbf\u95ee Service IP \u8bbf\u95ee\u5176\u5b83 Pod \u65f6\uff0c\u4f1a\u6784\u9020\u4e00\u4e2a\u76ee\u7684\u5730\u5740\u4e3a Service IP\u3001\u76ee\u7684 MAC \u5730\u5740\u4e3a\u7f51\u5173 MAC \u5730\u5740\u7684\u7f51\u7edc\u5305\u3002 \u7f51\u7edc\u5305\u8fdb\u5165\u903b\u8f91\u4ea4\u6362\u673a\u540e\uff0c\u8d1f\u8f7d\u5747\u8861\u4f1a\u5bf9\u7f51\u7edc\u5305\u8fdb\u884c\u62e6\u622a\u548c DNAT \u5904\u7406\uff0c\u5c06\u76ee\u7684 IP \u548c\u7aef\u53e3\u4fee\u6539\u4e3a Service \u5bf9\u5e94\u7684\u67d0\u4e2a Endpoint \u7684 IP \u548c\u7aef\u53e3\u3002 \u7531\u4e8e\u903b\u8f91\u4ea4\u6362\u673a\u5e76\u672a\u4fee\u6539\u7f51\u7edc\u5305\u7684\u4e8c\u5c42\u76ee\u7684 MAC \u5730\u5740\uff0c\u7f51\u7edc\u5305\u5728\u8fdb\u5165\u5916\u90e8\u4ea4\u6362\u673a\u540e\u4ecd\u7136\u4f1a\u9001\u5230\u5916\u90e8\u7f51\u5173\uff0c\u6b64\u65f6\u9700\u8981\u5916\u90e8\u7f51\u5173\u5bf9\u7f51\u7edc\u5305\u8fdb\u884c\u8f6c\u53d1\u3002</p>"},{"location":"reference/underlay-topology/#service-pod","title":"Service \u540e\u7aef\u4e3a\u540c\u8282\u70b9\u540c\u5b50\u7f51 Pod","text":""},{"location":"reference/underlay-topology/#service-pod_1","title":"Service \u540e\u7aef\u4e3a\u540c\u8282\u70b9\u4e0d\u540c\u5b50\u7f51 Pod","text":"<p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"start/one-step-install/","title":"\u4e00\u952e\u5b89\u88c5","text":"<p>Kube-OVN \u63d0\u4f9b\u4e86\u4e00\u952e\u5b89\u88c5\u811a\u672c\uff0c\u53ef\u4ee5\u5e2e\u52a9\u4f60\u5feb\u901f\u5b89\u88c5\u4e00\u4e2a\u9ad8\u53ef\u7528\uff0c\u751f\u4ea7\u5c31\u7eea\u7684 Kube-OVN \u5bb9\u5668\u7f51\u7edc\uff0c\u9ed8\u8ba4\u90e8\u7f72\u4e3a Overlay \u7c7b\u578b\u7f51\u7edc\u3002</p> <p>\u4ece Kube-OVN v1.12.0 \u7248\u672c\u5f00\u59cb\uff0c\u652f\u6301 Helm Chart \u5b89\u88c5\uff0c\u9ed8\u8ba4\u90e8\u7f72\u4e3a Overlay \u7c7b\u578b\u7f51\u7edc\u3002</p> <p>\u5982\u679c\u9ed8\u8ba4\u7f51\u7edc\u9700\u8981\u642d\u5efa Underlay/Vlan \u7f51\u7edc\uff0c\u8bf7\u53c2\u8003 Underlay \u7f51\u7edc\u652f\u6301\u3002</p> <p>\u5b89\u88c5\u524d\u8bf7\u53c2\u8003\u51c6\u5907\u5de5\u4f5c\u786e\u8ba4\u73af\u5883\u914d\u7f6e\u6b63\u786e\u3002</p>"},{"location":"start/one-step-install/#_2","title":"\u811a\u672c\u5b89\u88c5","text":""},{"location":"start/one-step-install/#_3","title":"\u4e0b\u8f7d\u5b89\u88c5\u811a\u672c","text":"<p>\u6211\u4eec\u63a8\u8350\u5728\u751f\u4ea7\u73af\u5883\u4f7f\u7528\u7a33\u5b9a\u7684 release \u7248\u672c\uff0c\u8bf7\u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u4e0b\u8f7d\u7a33\u5b9a\u7248\u672c\u5b89\u88c5\u811a\u672c\uff1a</p> <pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/images/install.sh\n</code></pre> <p>\u5982\u679c\u5bf9 master \u5206\u652f\u7684\u6700\u65b0\u529f\u80fd\u611f\u5174\u8da3\uff0c\u60f3\u4f7f\u7528\u4e0b\u9762\u7684\u547d\u4ee4\u4e0b\u8f7d\u5f00\u53d1\u7248\u672c\u90e8\u7f72\u811a\u672c\uff1a</p> <pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/images/install.sh\n</code></pre>"},{"location":"start/one-step-install/#_4","title":"\u4fee\u6539\u914d\u7f6e\u53c2\u6570","text":"<p>\u4f7f\u7528\u7f16\u8f91\u5668\u6253\u5f00\u811a\u672c\uff0c\u5e76\u4fee\u6539\u4e0b\u5217\u53d8\u91cf\u4e3a\u9884\u671f\u503c\uff1a</p> <pre><code>REGISTRY=\"kubeovn\"                     # \u955c\u50cf\u4ed3\u5e93\u5730\u5740\nVERSION=\"v1.13.0\"                      # \u955c\u50cf\u7248\u672c/Tag\nPOD_CIDR=\"10.16.0.0/16\"                # \u9ed8\u8ba4\u5b50\u7f51 CIDR \u4e0d\u8981\u548c SVC/NODE/JOIN CIDR \u91cd\u53e0\nSVC_CIDR=\"10.96.0.0/12\"                # \u9700\u8981\u548c apiserver \u7684 service-cluster-ip-range \u4fdd\u6301\u4e00\u81f4\nJOIN_CIDR=\"100.64.0.0/16\"              # Pod \u548c\u4e3b\u673a\u901a\u4fe1\u7f51\u7edc CIDR\uff0c\u4e0d\u8981\u548c SVC/NODE/POD CIDR \u91cd\u53e0 \nLABEL=\"node-role.kubernetes.io/master\" # \u90e8\u7f72 OVN DB \u8282\u70b9\u7684\u6807\u7b7e\nIFACE=\"\"                               # \u5bb9\u5668\u7f51\u7edc\u6240\u4f7f\u7528\u7684\u7684\u5bbf\u4e3b\u673a\u7f51\u5361\u540d\uff0c\u5982\u679c\u4e3a\u7a7a\u5219\u4f7f\u7528 Kubernetes \u4e2d\u7684 Node IP \u6240\u5728\u7f51\u5361\nTUNNEL_TYPE=\"geneve\"                   # \u96a7\u9053\u5c01\u88c5\u534f\u8bae\uff0c\u53ef\u9009 geneve, vxlan \u6216 stt\uff0cstt \u9700\u8981\u5355\u72ec\u7f16\u8bd1 ovs \u5185\u6838\u6a21\u5757\n</code></pre> <p>\u53ef\u4f7f\u7528\u6b63\u5219\u8868\u8fbe\u5f0f\u6765\u5339\u914d\u7f51\u5361\u540d\uff0c\u4f8b\u5982 <code>IFACE=enp6s0f0,eth.*</code>\u3002</p>"},{"location":"start/one-step-install/#_5","title":"\u6267\u884c\u5b89\u88c5\u811a\u672c","text":"<p><code>bash install.sh</code></p> <p>\u7b49\u5f85\u5b89\u88c5\u5b8c\u6210\u3002</p>"},{"location":"start/one-step-install/#helm-chart","title":"Helm Chart \u5b89\u88c5","text":"<p>\u7531\u4e8e Kube-OVN \u7684\u5b89\u88c5\uff0c\u9700\u8981\u8bbe\u7f6e\u4e00\u4e9b\u53c2\u6570\uff0c\u56e0\u6b64\u4f7f\u7528 Helm \u5b89\u88c5 Kube-OVN\uff0c\u9700\u8981\u6309\u7167\u4ee5\u4e0b\u6b65\u9aa4\u6267\u884c\u3002</p>"},{"location":"start/one-step-install/#ip","title":"\u67e5\u770b\u8282\u70b9 IP \u5730\u5740","text":"<pre><code>$ kubectl get node -o wide\nNAME                     STATUS     ROLES           AGE   VERSION   INTERNAL-IP   EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION      CONTAINER-RUNTIME\nkube-ovn-control-plane   NotReady   control-plane   20h   v1.26.0   172.18.0.3    &lt;none&gt;        Ubuntu 22.04.1 LTS   5.10.104-linuxkit   containerd://1.6.9\nkube-ovn-worker          NotReady   &lt;none&gt;          20h   v1.26.0   172.18.0.2    &lt;none&gt;        Ubuntu 22.04.1 LTS   5.10.104-linuxkit   containerd://1.6.9\n</code></pre>"},{"location":"start/one-step-install/#master","title":"\u53bb\u6389\u96c6\u7fa4 master \u8282\u70b9\u6c61\u70b9","text":"<pre><code>$ kubectl taint node kube-ovn-control-plane node-role.kubernetes.io/control-plane:NoSchedule-\nnode/kube-ovn-control-plane untainted\n</code></pre> <p>\u5982\u679c\u786e\u5b9a\u4e0d\u9700\u8981\u5728 master \u8282\u70b9\u8c03\u5ea6\u4e1a\u52a1 Pod\uff0c\u8fd9\u4e00\u6b65\u53ef\u4ee5\u8df3\u8fc7\u3002</p>"},{"location":"start/one-step-install/#label","title":"\u7ed9\u8282\u70b9\u6dfb\u52a0 label","text":"<pre><code>$ kubectl label node -lbeta.kubernetes.io/os=linux kubernetes.io/os=linux --overwrite\nnode/kube-ovn-control-plane not labeled\nnode/kube-ovn-worker not labeled\n\n$ kubectl label node -lnode-role.kubernetes.io/control-plane kube-ovn/role=master --overwrite\nnode/kube-ovn-control-plane labeled\n\n# \u4ee5\u4e0b label \u7528\u4e8e dpdk \u955c\u50cf\u7684\u5b89\u88c5\uff0c\u975e dpdk \u60c5\u51b5\uff0c\u53ef\u4ee5\u5ffd\u7565\n$ kubectl label node -lovn.kubernetes.io/ovs_dp_type!=userspace ovn.kubernetes.io/ovs_dp_type=kernel --overwrite\nnode/kube-ovn-control-plane labeled\nnode/kube-ovn-worker labeled\n</code></pre>"},{"location":"start/one-step-install/#helm-repo","title":"\u6dfb\u52a0 Helm Repo \u4fe1\u606f","text":"<pre><code>$ helm repo add kubeovn https://kubeovn.github.io/kube-ovn/\n\"kubeovn\" has been added to your repositories\n\n$ helm repo list\nNAME            URL\nkubeovn         https://kubeovn.github.io/kube-ovn/\n\n$ helm search repo kubeovn\nNAME                CHART VERSION   APP VERSION DESCRIPTION\nkubeovn/kube-ovn    0.1.0           1.12.0      Helm chart for Kube-OVN\n</code></pre>"},{"location":"start/one-step-install/#helm-install-kube-ovn","title":"\u6267\u884c helm install \u5b89\u88c5 Kube-OVN","text":"<p>Node0IP\u3001Node1IP\u3001Node2IP \u53c2\u6570\u5206\u522b\u4e3a\u96c6\u7fa4 master \u8282\u70b9\u7684 IP \u5730\u5740\u3002\u5176\u4ed6\u53c2\u6570\u7684\u8bbe\u7f6e\uff0c\u53ef\u4ee5\u53c2\u8003 values.yaml \u6587\u4ef6\u4e2d\u53d8\u91cf\u5b9a\u4e49\u3002</p> <pre><code># \u5355 master \u8282\u70b9\u73af\u5883\u5b89\u88c5\n$ helm install kube-ovn kubeovn/kube-ovn --set MASTER_NODES=${Node0IP}\n\n# \u4ee5\u4e0a\u8fb9\u7684 node \u4fe1\u606f\u4e3a\u4f8b\uff0c\u6267\u884c\u5b89\u88c5\u547d\u4ee4\n$ helm install kube-ovn kubeovn/kube-ovn --set MASTER_NODES=172.18.0.3\nNAME: kube-ovn\nLAST DEPLOYED: Fri Mar 31 12:43:43 2023\nNAMESPACE: default\nSTATUS: deployed\nREVISION: 1\nTEST SUITE: None\n\n# \u9ad8\u53ef\u7528\u96c6\u7fa4\u5b89\u88c5\n$ helm install kube-ovn kubeovn/kube-ovn --set MASTER_NODES=${Node0IP},${Node1IP},${Node2IP}, --set replicaCount=3\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"start/prepare/","title":"\u51c6\u5907\u5de5\u4f5c","text":"<p>Kube-OVN \u662f\u4e00\u4e2a\u7b26\u5408 CNI \u89c4\u8303\u7684\u7f51\u7edc\u7ec4\u4ef6\uff0c\u5176\u8fd0\u884c\u9700\u8981\u4f9d\u8d56 Kubernetes \u73af\u5883\u53ca\u5bf9\u5e94\u7684\u5185\u6838\u7f51\u7edc\u6a21\u5757\u3002 \u4ee5\u4e0b\u662f\u901a\u8fc7\u6d4b\u8bd5\u7684\u64cd\u4f5c\u7cfb\u7edf\u548c\u8f6f\u4ef6\u7248\u672c\uff0c\u73af\u5883\u914d\u7f6e\u548c\u6240\u9700\u8981\u5f00\u653e\u7684\u7aef\u53e3\u4fe1\u606f\u3002</p>"},{"location":"start/prepare/#_2","title":"\u8f6f\u4ef6\u7248\u672c","text":"<ul> <li>Kubernetes &gt;= 1.23\u3002</li> <li>Docker &gt;= 1.12.6, Containerd &gt;= 1.3.4\u3002</li> <li>\u64cd\u4f5c\u7cfb\u7edf: CentOS 7/8, Ubuntu 16.04/18.04/20.04\u3002</li> <li>\u5176\u4ed6 Linux \u53d1\u884c\u7248\uff0c\u9700\u8981\u68c0\u67e5\u4e00\u4e0b\u5185\u6838\u6a21\u5757\u662f\u5426\u5b58\u5728 <code>geneve</code>, <code>openvswitch</code>, <code>ip_tables</code> \u548c <code>iptable_nat</code>\uff0cKube-OVN \u6b63\u5e38\u5de5\u4f5c\u4f9d\u8d56\u4e0a\u8ff0\u6a21\u5757\u3002</li> </ul> <p>\u6ce8\u610f\u4e8b\u9879\uff1a</p> <ol> <li>\u5982\u679c\u5185\u6838\u7248\u672c\u4e3a 3.10.0-862 \u5185\u6838 <code>netfilter</code> \u6a21\u5757\u5b58\u5728 bug \u4f1a\u5bfc\u81f4 Kube-OVN \u5185\u7f6e\u8d1f\u8f7d\u5747\u8861\u5668\u65e0\u6cd5\u5de5\u4f5c\uff0c\u9700\u8981\u5bf9\u5185\u6838\u5347\u7ea7\uff0c\u5efa\u8bae\u4f7f\u7528 CentOS \u5b98\u65b9\u5bf9\u5e94\u7248\u672c\u6700\u65b0\u5185\u6838\u4fdd\u8bc1\u7cfb\u7edf\u7684\u5b89\u5168\u3002\u76f8\u5173\u5185\u6838 bug \u53c2\u8003 Floating IPs broken after kernel upgrade to Centos/RHEL 7.5 - DNAT not working\u3002</li> <li>Rocky Linux 8.6 \u7684\u5185\u6838 4.18.0-372.9.1.el8.x86_64 \u5b58\u5728 TCP \u901a\u4fe1\u95ee\u9898 TCP connection failed in Rocky Linux 8.6\uff0c\u8bf7\u5347\u7ea7\u5185\u6838\u81f3 4.18.0-372.13.1.el8_6.x86_64 \u6216\u66f4\u9ad8\u7248\u672c\u3002</li> <li>\u5982\u679c\u5185\u6838\u7248\u672c\u4e3a 4.4 \u5219\u5bf9\u5e94\u7684\u5185\u6838 <code>openvswitch</code> \u6a21\u5757\u5b58\u5728\u95ee\u9898\uff0c\u5efa\u8bae\u5347\u7ea7\u6216\u624b\u52a8\u7f16\u8bd1 <code>openvswitch</code> \u65b0\u7248\u672c\u6a21\u5757\u8fdb\u884c\u66f4\u65b0</li> <li>Geneve \u96a7\u9053\u5efa\u7acb\u9700\u8981\u68c0\u67e5 IPv6\uff0c\u53ef\u901a\u8fc7 <code>cat /proc/cmdline</code> \u68c0\u67e5\u5185\u6838\u542f\u52a8\u53c2\u6570\uff0c \u76f8\u5173\u5185\u6838 bug \u8bf7\u53c2\u8003 Geneve tunnels don't work when ipv6 is disabled\u3002</li> </ol>"},{"location":"start/prepare/#_3","title":"\u73af\u5883\u914d\u7f6e","text":"<ul> <li>Kernel \u542f\u52a8\u9700\u8981\u5f00\u542f ipv6, \u5982\u679c kernel \u542f\u52a8\u53c2\u6570\u5305\u542b <code>ipv6.disable=1</code> \u9700\u8981\u5c06\u5176\u8bbe\u7f6e\u4e3a 0\u3002</li> <li><code>kube-proxy</code> \u6b63\u5e38\u5de5\u4f5c\uff0cKube-OVN \u53ef\u4ee5\u901a\u8fc7 SVC IP \u8bbf\u95ee\u5230 <code>kube-apiserver</code>\u3002</li> <li>\u786e\u8ba4 kubelet \u914d\u7f6e\u53c2\u6570\u5f00\u542f\u4e86 CNI\uff0c\u5e76\u4e14\u914d\u7f6e\u5728\u6807\u51c6\u8def\u5f84\u4e0b, kubelet \u542f\u52a8\u65f6\u5e94\u5305\u542b\u5982\u4e0b\u53c2\u6570 <code>--network-plugin=cni --cni-bin-dir=/opt/cni/bin --cni-conf-dir=/etc/cni/net.d</code>\u3002</li> <li>\u786e\u8ba4\u672a\u5b89\u88c5\u5176\u4ed6\u7f51\u7edc\u63d2\u4ef6\uff0c\u6216\u8005\u5176\u4ed6\u7f51\u7edc\u63d2\u4ef6\u5df2\u7ecf\u88ab\u6e05\u9664\uff0c\u68c0\u67e5 <code>/etc/cni/net.d/</code> \u8def\u5f84\u4e0b\u65e0\u5176\u4ed6\u7f51\u7edc\u63d2\u4ef6\u914d\u7f6e\u6587\u4ef6\u3002\u5982\u679c\u4e4b\u524d\u5b89\u88c5\u8fc7\u5176\u4ed6\u7f51\u7edc\u63d2\u4ef6\uff0c\u5efa\u8bae\u5220\u9664\u540e\u91cd\u542f\u673a\u5668\u6e05\u7406\u6b8b\u7559\u7f51\u7edc\u8d44\u6e90\u3002</li> </ul>"},{"location":"start/prepare/#_4","title":"\u7aef\u53e3\u4fe1\u606f","text":"\u7ec4\u4ef6 \u7aef\u53e3 \u7528\u9014 ovn-central 6641/tcp, 6642/tcp, 6643/tcp, 6644/tcp ovn-db \u548c raft server \u76d1\u542c\u7aef\u53e3 ovs-ovn Geneve 6081/udp, STT 7471/tcp, Vxlan 4789/udp \u96a7\u9053\u7aef\u53e3 kube-ovn-controller 10660/tcp \u76d1\u63a7\u76d1\u542c\u7aef\u53e3 kube-ovn-daemon 10665/tcp \u76d1\u63a7\u76d1\u542c\u7aef\u53e3 kube-ovn-monitor 10661/tcp \u76d1\u63a7\u76d1\u542c\u7aef\u53e3 <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"start/sealos-install/","title":"\u4f7f\u7528 sealos \u4e00\u952e\u90e8\u7f72 Kubernetes \u548c Kube-OVN","text":"<p>sealos \u4f5c\u4e3a Kubernetes \u7684\u4e00\u4e2a\u53d1\u884c\u7248\uff0c\u901a\u8fc7\u6781\u7b80\u7684\u4f7f\u7528\u65b9\u5f0f\u548c\u56fd\u5185\u7684\u955c\u50cf\u4ed3\u5e93\uff0c\u53ef\u4ee5\u5e2e\u52a9\u7528\u6237\u5feb\u901f\u4ece\u96f6\u521d\u59cb\u5316\u4e00\u4e2a\u5bb9\u5668\u96c6\u7fa4\u3002 \u901a\u8fc7\u4f7f\u7528 sealos \u7528\u6237\u53ef\u4ee5\u901a\u8fc7\u4e00\u6761\u547d\u4ee4\u5728\u51e0\u5206\u949f\u5185\u90e8\u7f72\u51fa\u4e00\u4e2a\u5b89\u88c5\u597d Kube-OVN \u7684 Kubernetes \u96c6\u7fa4\u3002</p>"},{"location":"start/sealos-install/#sealos","title":"\u4e0b\u8f7d\u5b89\u88c5 sealos","text":"AMD64 ARM64 <pre><code>wget  https://github.com/labring/sealos/releases/download/v4.1.4/sealos_4.1.4_linux_amd64.tar.gz  &amp;&amp; \\\ntar -zxvf sealos_4.1.4_linux_amd64.tar.gz sealos &amp;&amp;  chmod +x sealos &amp;&amp; mv sealos /usr/bin\n</code></pre> <pre><code>wget  https://github.com/labring/sealos/releases/download/v4.1.4/sealos_4.1.4_linux_arm64.tar.gz  &amp;&amp; \\\ntar -zxvf sealos_4.1.4_linux_arm64.tar.gz sealos &amp;&amp;  chmod +x sealos &amp;&amp; mv sealos /usr/bin\n</code></pre>"},{"location":"start/sealos-install/#kubernetes-kube-ovn","title":"\u90e8\u7f72 Kubernetes \u548c Kube-OVN","text":"<pre><code>```bash\nsealos run labring/kubernetes:v1.24.3 labring/kube-ovn:v1.10.5 \\\n  --masters [masters ips seperated by comma] \\\n  --nodes [nodes ips seperated by comma] -p [your-ssh-passwd]\n```\n</code></pre>"},{"location":"start/sealos-install/#_1","title":"\u7b49\u5f85\u90e8\u7f72\u5b8c\u6210","text":"<pre><code>```bash\n[Step 6/6] Finish\n\n                    ,,,,\n                    ,::,\n                   ,,::,,,,\n            ,,,,,::::::::::::,,,,,\n         ,,,::::::::::::::::::::::,,,\n       ,,::::::::::::::::::::::::::::,,\n     ,,::::::::::::::::::::::::::::::::,,\n    ,::::::::::::::::::::::::::::::::::::,\n   ,:::::::::::::,,   ,,:::::,,,::::::::::,\n ,,:::::::::::::,       ,::,     ,:::::::::,\n ,:::::::::::::,   :x,  ,::  :,   ,:::::::::,\n,:::::::::::::::,  ,,,  ,::, ,,  ,::::::::::,\n,:::::::::::::::::,,,,,,:::::,,,,::::::::::::,    ,:,   ,:,            ,xx,                            ,:::::,   ,:,     ,:: :::,    ,x\n,::::::::::::::::::::::::::::::::::::::::::::,    :x: ,:xx:        ,   :xx,                          :xxxxxxxxx, :xx,   ,xx:,xxxx,   :x\n,::::::::::::::::::::::::::::::::::::::::::::,    :xxxxx:,  ,xx,  :x:  :xxx:x::,  ::xxxx:           :xx:,  ,:xxx  :xx, ,xx: ,xxxxx:, :x\n,::::::::::::::::::::::::::::::::::::::::::::,    :xxxxx,   :xx,  :x:  :xxx,,:xx,:xx:,:xx, ,,,,,,,,,xxx,    ,xx:   :xx:xx:  ,xxx,:xx::x\n,::::::,,::::::::,,::::::::,,:::::::,,,::::::,    :x:,xxx:  ,xx,  :xx  :xx:  ,xx,xxxxxx:, ,xxxxxxx:,xxx:,  ,xxx,    :xxx:   ,xxx, :xxxx\n,::::,    ,::::,   ,:::::,   ,,::::,    ,::::,    :x:  ,:xx,,:xx::xxxx,,xxx::xx: :xx::::x: ,,,,,,   ,xxxxxxxxx,     ,xx:    ,xxx,  :xxx\n,::::,    ,::::,    ,::::,    ,::::,    ,::::,    ,:,    ,:,  ,,::,,:,  ,::::,,   ,:::::,            ,,:::::,        ,,      :x:    ,::\n,::::,    ,::::,    ,::::,    ,::::,    ,::::,\n ,,,,,    ,::::,    ,::::,    ,::::,    ,:::,             ,,,,,,,,,,,,,\n          ,::::,    ,::::,    ,::::,    ,:::,        ,,,:::::::::::::::,\n          ,::::,    ,::::,    ,::::,    ,::::,  ,,,,:::::::::,,,,,,,:::,\n          ,::::,    ,::::,    ,::::,     ,::::::::::::,,,,,\n           ,,,,     ,::::,     ,,,,       ,,,::::,,,,\n                    ,::::,\n                    ,,::,\n\nThanks for choosing Kube-OVN!\nFor more advanced features, please read https://github.com/kubeovn/kube-ovn#documents\nIf you have any question, please file an issue https://github.com/kubeovn/kube-ovn/issues/new/choose\n2022-08-10T16:31:34 info succeeded in creating a new cluster, enjoy it!\n2022-08-10T16:31:34 info\n      ___           ___           ___           ___       ___           ___\n     /\\  \\         /\\  \\         /\\  \\         /\\__\\     /\\  \\         /\\  \\\n    /::\\  \\       /::\\  \\       /::\\  \\       /:/  /    /::\\  \\       /::\\  \\\n   /:/\\ \\  \\     /:/\\:\\  \\     /:/\\:\\  \\     /:/  /    /:/\\:\\  \\     /:/\\ \\  \\\n  _\\:\\~\\ \\  \\   /::\\~\\:\\  \\   /::\\~\\:\\  \\   /:/  /    /:/  \\:\\  \\   _\\:\\~\\ \\  \\\n /\\ \\:\\ \\ \\__\\ /:/\\:\\ \\:\\__\\ /:/\\:\\ \\:\\__\\ /:/__/    /:/__/ \\:\\__\\ /\\ \\:\\ \\ \\__\\\n \\:\\ \\:\\ \\/__/ \\:\\~\\:\\ \\/__/ \\/__\\:\\/:/  / \\:\\  \\    \\:\\  \\ /:/  / \\:\\ \\:\\ \\/__/\n  \\:\\ \\:\\__\\    \\:\\ \\:\\__\\        \\::/  /   \\:\\  \\    \\:\\  /:/  /   \\:\\ \\:\\__\\\n   \\:\\/:/  /     \\:\\ \\/__/        /:/  /     \\:\\  \\    \\:\\/:/  /     \\:\\/:/  /\n    \\::/  /       \\:\\__\\         /:/  /       \\:\\__\\    \\::/  /       \\::/  /\n     \\/__/         \\/__/         \\/__/         \\/__/     \\/__/         \\/__/\n\n                  Website :https://www.sealos.io/\n                  Address :github.com/labring/sealos\n```\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"start/underlay/","title":"Underlay \u7f51\u7edc\u5b89\u88c5","text":"<p>\u9ed8\u8ba4\u60c5\u51b5\u4e0b Kube-OVN \u7684\u9ed8\u8ba4\u5b50\u7f51\u4f7f\u7528 Geneve \u5bf9\u8de8\u4e3b\u673a\u6d41\u91cf\u8fdb\u884c\u5c01\u88c5\uff0c\u5728\u57fa\u7840\u8bbe\u65bd\u4e4b\u4e0a\u62bd\u8c61\u51fa\u4e00\u5c42\u865a\u62df\u7684 Overlay \u7f51\u7edc\u3002</p> <p>\u5bf9\u4e8e\u5e0c\u671b\u5bb9\u5668\u7f51\u7edc\u76f4\u63a5\u4f7f\u7528\u7269\u7406\u7f51\u7edc\u5730\u5740\u6bb5\u60c5\u51b5\uff0c\u53ef\u4ee5\u5c06 Kube-OVN \u7684\u9ed8\u8ba4\u5b50\u7f51\u5de5\u4f5c\u5728 Underlay \u6a21\u5f0f\uff0c\u53ef\u4ee5\u76f4\u63a5\u7ed9\u5bb9\u5668\u5206\u914d\u7269\u7406\u7f51\u7edc\u4e2d\u7684\u5730\u5740\u8d44\u6e90\uff0c\u8fbe\u5230\u66f4\u597d\u7684\u6027\u80fd\u4ee5\u53ca\u548c\u7269\u7406\u7f51\u7edc\u7684\u8fde\u901a\u6027\u3002</p> <p></p>"},{"location":"start/underlay/#_1","title":"\u529f\u80fd\u9650\u5236","text":"<p>\u7531\u4e8e\u8be5\u6a21\u5f0f\u4e0b\u5bb9\u5668\u7f51\u7edc\u76f4\u63a5\u4f7f\u7528\u7269\u7406\u7f51\u7edc\u8fdb\u884c\u4e8c\u5c42\u5305\u8f6c\u53d1\uff0cOverlay \u6a21\u5f0f\u4e0b\u7684 SNAT/EIP\uff0c \u5206\u5e03\u5f0f\u7f51\u5173/\u96c6\u4e2d\u5f0f\u7f51\u5173\u7b49 L3 \u529f\u80fd\u65e0\u6cd5\u4f7f\u7528\uff0cVPC \u7ea7\u522b\u7684\u9694\u79bb\u4e5f\u65e0\u6cd5\u5bf9 Underlay \u5b50\u7f51\u751f\u6548\u3002</p>"},{"location":"start/underlay/#macvlan","title":"\u548c Macvlan \u6bd4\u8f83","text":"<p>Kube-OVN \u7684 Underlay \u6a21\u5f0f\u548c Macvlan \u5de5\u4f5c\u6a21\u5f0f\u5341\u5206\u7c7b\u4f3c\uff0c\u5728\u529f\u80fd\u548c\u6027\u80fd\u4e0a\u4e3b\u8981\u6709\u4ee5\u4e0b\u51e0\u4e2a\u533a\u522b\uff1a</p> <ol> <li>\u7531\u4e8e Macvlan \u7684\u5185\u6838\u8def\u5f84\u66f4\u77ed\uff0c\u5e76\u4e14\u4e0d\u9700\u8981 OVS \u5bf9\u6570\u636e\u5305\u8fdb\u884c\u5904\u7406\uff0cMacvlan \u5728\u541e\u5410\u91cf\u548c\u5ef6\u8fdf\u6027\u80fd\u6307\u6807\u4e0a\u8868\u73b0\u4f1a\u66f4\u597d\u3002</li> <li>Kube-OVN \u901a\u8fc7\u6d41\u8868\u63d0\u4f9b\u4e86 arp-proxy \u529f\u80fd\uff0c\u53ef\u4ee5\u7f13\u89e3\u5927\u89c4\u6a21\u7f51\u7edc\u4e0b\u7684 arp \u5e7f\u64ad\u98ce\u66b4\u98ce\u9669\u3002</li> <li>\u7531\u4e8e Macvlan \u5de5\u4f5c\u5728\u5185\u6838\u5e95\u5c42\uff0c\u4f1a\u7ed5\u8fc7\u5bbf\u4e3b\u673a\u7684 netfilter\uff0cService \u548c NetworkPolicy \u529f\u80fd\u9700\u8981\u989d\u5916\u5f00\u53d1\u3002Kube-OVN \u901a\u8fc7 OVS \u6d41\u8868\u63d0\u4f9b\u4e86 Service \u548c NetworkPolicy \u7684\u80fd\u529b\u3002</li> <li>Kube-OVN \u7684 Underlay \u6a21\u5f0f\u76f8\u6bd4 Macvlan \u989d\u5916\u63d0\u4f9b\u4e86\u5730\u5740\u7ba1\u7406\uff0c\u56fa\u5b9a IP \u548c QoS \u7b49\u529f\u80fd\u3002</li> </ol>"},{"location":"start/underlay/#_2","title":"\u73af\u5883\u8981\u6c42","text":"<p>\u5728 Underlay \u6a21\u5f0f\u4e0b\uff0cOVS \u5c06\u4f1a\u6865\u63a5\u4e00\u4e2a\u8282\u70b9\u7f51\u5361\u5230 OVS \u7f51\u6865\uff0c\u5e76\u5c06\u6570\u636e\u5305\u76f4\u63a5\u901a\u8fc7\u8be5\u8282\u70b9\u7f51\u5361\u5bf9\u5916\u53d1\u9001\uff0cL2/L3 \u5c42\u9762\u7684\u8f6c\u53d1\u80fd\u529b\u9700\u8981\u4f9d\u8d56\u5e95\u5c42\u7f51\u7edc\u8bbe\u5907\u3002 \u9700\u8981\u9884\u5148\u5728\u5e95\u5c42\u7f51\u7edc\u8bbe\u5907\u914d\u7f6e\u5bf9\u5e94\u7684\u7f51\u5173\u3001Vlan \u548c\u5b89\u5168\u7b56\u7565\u7b49\u914d\u7f6e\u3002</p> <ol> <li>\u5bf9\u4e8e OpenStack \u7684 VM \u73af\u5883\uff0c\u9700\u8981\u5c06\u5bf9\u5e94\u7f51\u7edc\u7aef\u53e3\u7684 <code>PortSecurity</code> \u5173\u95ed\u3002</li> <li>\u5bf9\u4e8e VMware \u7684 vSwitch \u7f51\u7edc\uff0c\u9700\u8981\u5c06 <code>MAC Address Changes</code>, <code>Forged Transmits</code> \u548c <code>Promiscuous Mode Operation</code> \u8bbe\u7f6e\u4e3a <code>allow</code>\u3002</li> <li>\u5bf9\u4e8e Hyper-V \u865a\u62df\u5316\uff0c\u9700\u8981\u5f00\u542f\u865a\u62df\u673a\u7f51\u5361\u9ad8\u7ea7\u529f\u80fd\u4e2d\u7684 <code>MAC Address Spoofing</code>\u3002</li> <li>\u516c\u6709\u4e91\uff0c\u4f8b\u5982 AWS\u3001GCE\u3001\u963f\u91cc\u4e91\u7b49\u7531\u4e8e\u4e0d\u652f\u6301\u7528\u6237\u81ea\u5b9a\u4e49 Mac \u65e0\u6cd5\u652f\u6301 Underlay \u6a21\u5f0f\u7f51\u7edc\uff0c\u5728\u8fd9\u79cd\u573a\u666f\u4e0b\u5982\u679c\u60f3\u4f7f\u7528 Underlay \u63a8\u8350\u4f7f\u7528\u5bf9\u5e94\u516c\u6709\u4e91\u5382\u5546\u63d0\u4f9b\u7684 VPC-CNI\u3002</li> <li>\u6865\u63a5\u7f51\u5361\u4e0d\u80fd\u4e3a Linux Bridge\u3002</li> </ol> <p>\u5bf9\u4e8e\u7ba1\u7406\u7f51\u548c\u5bb9\u5668\u7f51\u4f7f\u7528\u540c\u4e00\u4e2a\u7f51\u5361\u7684\u60c5\u51b5\u4e0b\uff0cKube-OVN \u4f1a\u5c06\u7f51\u5361\u7684 Mac \u5730\u5740\u3001IP \u5730\u5740\u3001\u8def\u7531\u4ee5\u53ca MTU \u5c06\u8f6c\u79fb\u6216\u590d\u5236\u81f3\u5bf9\u5e94\u7684 OVS Bridge\uff0c \u4ee5\u652f\u6301\u5355\u7f51\u5361\u90e8\u7f72 Underlay \u7f51\u7edc\u3002OVS Bridge \u540d\u79f0\u683c\u5f0f\u4e3a <code>br-PROVIDER_NAME</code>\uff0c<code>PROVIDER_NAME</code> \u4e3a Provider \u7f51\u7edc\u540d\u79f0\uff08\u9ed8\u8ba4\u4e3a provider\uff09\u3002</p>"},{"location":"start/underlay/#_3","title":"\u90e8\u7f72\u65f6\u6307\u5b9a\u7f51\u7edc\u6a21\u5f0f","text":"<p>\u8be5\u90e8\u7f72\u6a21\u5f0f\u5c06\u9ed8\u8ba4\u5b50\u7f51\u8bbe\u7f6e\u4e3a Underlay \u6a21\u5f0f\uff0c\u6240\u6709\u672a\u6307\u5b9a\u5b50\u7f51\u7684 Pod \u5747\u4f1a\u9ed8\u8ba4\u8fd0\u884c\u5728 Underlay \u7f51\u7edc\u4e2d\u3002</p>"},{"location":"start/underlay/#_4","title":"\u4e0b\u8f7d\u5b89\u88c5\u811a\u672c","text":"<pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/images/install.sh\n</code></pre>"},{"location":"start/underlay/#_5","title":"\u4fee\u6539\u811a\u672c\u4e2d\u76f8\u5e94\u914d\u7f6e","text":"<pre><code>ENABLE_ARP_DETECT_IP_CONFLICT # \u5982\u6709\u9700\u8981\uff0c\u53ef\u4ee5\u9009\u62e9\u5173\u95ed vlan \u7f51\u7edc arp \u51b2\u7a81\u68c0\u6d4b\nNETWORK_TYPE                  # \u8bbe\u7f6e\u4e3a vlan\nVLAN_INTERFACE_NAME           # \u8bbe\u7f6e\u4e3a\u5bbf\u4e3b\u673a\u4e0a\u627f\u62c5\u5bb9\u5668\u6d41\u91cf\u7684\u7f51\u5361\uff0c\u4f8b\u5982 eth1\nVLAN_ID                       # \u4ea4\u6362\u673a\u6240\u63a5\u53d7\u7684 VLAN Tag\uff0c\u82e5\u8bbe\u7f6e\u4e3a 0 \u5219\u4e0d\u505a VLAN \u5c01\u88c5\nPOD_CIDR                      # \u8bbe\u7f6e\u4e3a\u7269\u7406\u7f51\u7edc CIDR\uff0c \u4f8b\u5982 192.168.1.0/24\nPOD_GATEWAY                   # \u8bbe\u7f6e\u4e3a\u7269\u7406\u7f51\u7edc\u7f51\u5173\uff0c\u4f8b\u5982 192.168.1.1\nEXCLUDE_IPS                   # \u6392\u9664\u8303\u56f4\uff0c\u907f\u514d\u5bb9\u5668\u7f51\u6bb5\u548c\u7269\u7406\u7f51\u7edc\u5df2\u7528 IP \u51b2\u7a81\uff0c\u4f8b\u5982 192.168.1.1..192.168.1.100\nENABLE_LB                     # \u5982\u679c Underlay \u5b50\u7f51\u9700\u8981\u4f7f\u7528 Service \u9700\u8981\u8bbe\u7f6e\u4e3a true \nEXCHANGE_LINK_NAME            # \u662f\u5426\u4ea4\u6362\u9ed8\u8ba4 provider-network \u4e0b OVS \u7f51\u6865\u548c\u6865\u63a5\u7f51\u5361\u7684\u540d\u5b57\uff0c\u9ed8\u8ba4\u4e3a false\nLS_DNAT_MOD_DL_DST            # DNAT \u65f6\u662f\u5426\u5bf9 MAC \u5730\u5740\u8fdb\u884c\u8f6c\u6362\uff0c\u53ef\u52a0\u901f Service \u7684\u8bbf\u95ee\uff0c\u9ed8\u8ba4\u4e3a true\n</code></pre>"},{"location":"start/underlay/#_6","title":"\u8fd0\u884c\u5b89\u88c5\u811a\u672c","text":"<pre><code>bash install.sh\n</code></pre>"},{"location":"start/underlay/#crd-underlay","title":"\u901a\u8fc7 CRD \u52a8\u6001\u521b\u5efa Underlay \u7f51\u7edc","text":"<p>\u8be5\u65b9\u5f0f\u53ef\u5728\u5b89\u88c5\u540e\u52a8\u6001\u7684\u521b\u5efa\u67d0\u4e2a Underlay \u5b50\u7f51\u4f9b Pod \u4f7f\u7528\u3002\u9700\u8981\u914d\u7f6e <code>ProviderNetwork</code>\uff0c<code>Vlan</code> \u548c <code>Subnet</code> \u4e09\u79cd\u81ea\u5b9a\u4e49\u8d44\u6e90\u3002</p>"},{"location":"start/underlay/#providernetwork","title":"\u521b\u5efa ProviderNetwork","text":"<p>ProviderNetwork \u63d0\u4f9b\u4e86\u4e3b\u673a\u7f51\u5361\u5230\u7269\u7406\u7f51\u7edc\u6620\u5c04\u7684\u62bd\u8c61\uff0c\u5c06\u540c\u5c5e\u4e00\u4e2a\u7f51\u7edc\u7684\u7f51\u5361\u8fdb\u884c\u7edf\u4e00\u7ba1\u7406\uff0c \u5e76\u89e3\u51b3\u5728\u590d\u6742\u73af\u5883\u4e0b\u540c\u673a\u5668\u591a\u7f51\u5361\u3001\u7f51\u5361\u540d\u4e0d\u4e00\u81f4\u3001\u5bf9\u5e94 Underlay \u7f51\u7edc\u4e0d\u4e00\u81f4\u7b49\u60c5\u51b5\u4e0b\u7684\u914d\u7f6e\u95ee\u9898\u3002</p> <p>\u521b\u5efa\u5982\u4e0b ProviderNetwork \u5e76\u5e94\u7528:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: ProviderNetwork\nmetadata:\n  name: net1\nspec:\n  defaultInterface: eth1\n  customInterfaces:\n    - interface: eth2\n      nodes:\n        - node1\n  excludeNodes:\n    - node2\n</code></pre> <p>\u6ce8\u610f\uff1aProviderNetwork \u8d44\u6e90\u540d\u79f0\u7684\u957f\u5ea6\u4e0d\u5f97\u8d85\u8fc7 12\u3002</p> <ul> <li><code>defaultInterface</code>: \u4e3a\u9ed8\u8ba4\u4f7f\u7528\u7684\u8282\u70b9\u7f51\u5361\u540d\u79f0\u3002 ProviderNetwork \u521b\u5efa\u6210\u529f\u540e\uff0c\u5404\u8282\u70b9\uff08\u9664 excludeNodes \u5916\uff09\u4e2d\u4f1a\u521b\u5efa\u540d\u4e3a br-net1\uff08\u683c\u5f0f\u4e3a <code>br-NAME</code>\uff09\u7684 OVS \u7f51\u6865\uff0c\u5e76\u5c06\u6307\u5b9a\u7684\u8282\u70b9\u7f51\u5361\u6865\u63a5\u81f3\u6b64\u7f51\u6865\u3002</li> <li><code>customInterfaces</code>: \u4e3a\u53ef\u9009\u9879\uff0c\u53ef\u9488\u5bf9\u7279\u5b9a\u8282\u70b9\u6307\u5b9a\u9700\u8981\u4f7f\u7528\u7684\u7f51\u5361\u3002</li> <li><code>excludeNodes</code>: \u53ef\u9009\u9879\uff0c\u7528\u4e8e\u6307\u5b9a\u4e0d\u6865\u63a5\u7f51\u5361\u7684\u8282\u70b9\u3002\u8be5\u5217\u8868\u4e2d\u7684\u8282\u70b9\u4f1a\u88ab\u6dfb\u52a0 <code>net1.provider-network.ovn.kubernetes.io/exclude=true</code> \u6807\u7b7e\u3002</li> </ul> <p>\u5176\u5b83\u8282\u70b9\u4f1a\u88ab\u6dfb\u52a0\u5982\u4e0b\u6807\u7b7e\uff1a</p> Key Value \u63cf\u8ff0 net1.provider-network.ovn.kubernetes.io/ready true \u8282\u70b9\u4e2d\u7684\u6865\u63a5\u5de5\u4f5c\u5df2\u5b8c\u6210\uff0cProviderNetwork \u5728\u8282\u70b9\u4e2d\u53ef\u7528 net1.provider-network.ovn.kubernetes.io/interface eth1 \u8282\u70b9\u4e2d\u88ab\u6865\u63a5\u7684\u7f51\u5361\u7684\u540d\u79f0 net1.provider-network.ovn.kubernetes.io/mtu 1500 \u8282\u70b9\u4e2d\u88ab\u6865\u63a5\u7684\u7f51\u5361\u7684 MTU <p>\u5982\u679c\u8282\u70b9\u7f51\u5361\u4e0a\u5df2\u7ecf\u914d\u7f6e\u4e86 IP\uff0c\u5219 IP \u5730\u5740\u548c\u7f51\u5361\u4e0a\u7684\u8def\u7531\u4f1a\u88ab\u8f6c\u79fb\u81f3\u5bf9\u5e94\u7684 OVS \u7f51\u6865\u3002</p>"},{"location":"start/underlay/#vlan","title":"\u521b\u5efa VLAN","text":"<p>Vlan \u63d0\u4f9b\u4e86\u5c06 Vlan Tag \u548c ProviderNetwork \u8fdb\u884c\u7ed1\u5b9a\u7684\u80fd\u529b\u3002</p> <p>\u521b\u5efa\u5982\u4e0b VLAN \u5e76\u5e94\u7528\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Vlan\nmetadata:\n  name: vlan1\nspec:\n  id: 0\n  provider: net1\n</code></pre> <ul> <li><code>id</code>: \u4e3a VLAN ID/Tag\uff0cKube-OVN \u4f1a\u5bf9\u5bf9\u8be5 Vlan \u4e0b\u7684\u6d41\u91cf\u589e\u52a0 Vlan \u6807\u7b7e\uff0c\u4e3a 0 \u65f6\u4e0d\u589e\u52a0\u4efb\u4f55\u6807\u7b7e\u3002</li> <li><code>provider</code>: \u4e3a\u9700\u8981\u4f7f\u7528\u7684 ProviderNetwork \u8d44\u6e90\u7684\u540d\u79f0\u3002\u591a\u4e2a VLAN \u53ef\u4ee5\u5f15\u7528\u540c\u4e00\u4e2a ProviderNetwork\u3002</li> </ul>"},{"location":"start/underlay/#subnet","title":"\u521b\u5efa Subnet","text":"<p>\u5c06 Vlan \u548c\u4e00\u4e2a\u5b50\u7f51\u7ed1\u5b9a\uff0c\u5982\u4e0b\u6240\u793a\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: subnet1\nspec:\nprotocol: IPv4\ncidrBlock: 172.17.0.0/16\ngateway: 172.17.0.1\nvlan: vlan1\n</code></pre> <p>\u5c06 <code>vlan</code> \u7684\u503c\u6307\u5b9a\u4e3a\u9700\u8981\u4f7f\u7528\u7684 VLAN \u540d\u79f0\u5373\u53ef\u3002\u591a\u4e2a Subnet \u53ef\u4ee5\u5f15\u7528\u540c\u4e00\u4e2a VLAN\u3002</p>"},{"location":"start/underlay/#_7","title":"\u5bb9\u5668\u521b\u5efa","text":"<p>\u53ef\u6309\u6b63\u5e38\u5bb9\u5668\u521b\u5efa\u65b9\u5f0f\u8fdb\u884c\u521b\u5efa\uff0c\u67e5\u770b\u5bb9\u5668 IP \u662f\u5426\u5728\u89c4\u5b9a\u8303\u56f4\u5185\uff0c\u4ee5\u53ca\u5bb9\u5668\u662f\u5426\u53ef\u4ee5\u548c\u7269\u7406\u7f51\u7edc\u4e92\u901a\u3002</p> <p>\u5982\u6709\u56fa\u5b9a IP \u9700\u6c42\uff0c\u53ef\u53c2\u8003 Pod \u56fa\u5b9a IP \u548c Mac</p>"},{"location":"start/underlay/#_8","title":"\u4f7f\u7528\u903b\u8f91\u7f51\u5173","text":"<p>\u5bf9\u4e8e\u7269\u7406\u7f51\u7edc\u4e0d\u5b58\u5728\u7f51\u5173\u7684\u60c5\u51b5\uff0cKube-OVN \u652f\u6301\u5728 Underlay \u6a21\u5f0f\u7684\u5b50\u7f51\u4e2d\u914d\u7f6e\u4f7f\u7528\u903b\u8f91\u7f51\u5173\u3002 \u82e5\u8981\u4f7f\u7528\u6b64\u529f\u80fd\uff0c\u8bbe\u7f6e\u5b50\u7f51\u7684 <code>spec.logicalGateway</code> \u4e3a <code>true</code> \u5373\u53ef\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n  name: subnet1\nspec:\n  protocol: IPv4\n  cidrBlock: 172.17.0.0/16\n  gateway: 172.17.0.1\n  vlan: vlan1\n  logicalGateway: true\n</code></pre> <p>\u5f00\u542f\u6b64\u529f\u80fd\u540e\uff0cPod \u4e0d\u4f7f\u7528\u5916\u90e8\u7f51\u5173\uff0c\u800c\u662f\u4f7f\u7528 Kube-OVN \u521b\u5efa\u7684\u903b\u8f91\u8def\u7531\u5668\uff08Logical Router\uff09\u5bf9\u4e8e\u8de8\u7f51\u6bb5\u901a\u4fe1\u8fdb\u884c\u8f6c\u53d1\u3002</p>"},{"location":"start/underlay/#underlay-overlay","title":"Underlay \u548c Overlay \u7f51\u7edc\u4e92\u901a","text":"<p>\u5982\u679c\u4e00\u4e2a\u96c6\u7fa4\u540c\u65f6\u5b58\u5728 Underlay \u548c Overlay \u5b50\u7f51\uff0c\u9ed8\u8ba4\u60c5\u51b5\u4e0b Overlay \u5b50\u7f51\u4e0b\u7684 Pod \u53ef\u4ee5\u901a\u8fc7\u7f51\u5173\u4ee5 NAT \u7684\u65b9\u5f0f\u8bbf\u95ee Underlay \u5b50\u7f51\u4e0b\u7684 Pod IP\u3002 \u5728 Underlay \u5b50\u7f51\u7684 Pod \u770b\u6765 Overlay \u5b50\u7f51\u7684\u5730\u5740\u662f\u4e00\u4e2a\u5916\u90e8\u7684\u5730\u5740\uff0c\u9700\u8981\u901a\u8fc7\u5e95\u5c42\u7269\u7406\u8bbe\u5907\u53bb\u8f6c\u53d1\uff0c\u4f46\u5e95\u5c42\u7269\u7406\u8bbe\u5907\u5e76\u4e0d\u6e05\u695a Overlay \u5b50\u7f51\u7684\u5730\u5740\u65e0\u6cd5\u8fdb\u884c\u8f6c\u53d1\u3002 \u56e0\u6b64 Underlay \u5b50\u7f51\u4e0b\u7684 Pod \u65e0\u6cd5\u901a\u8fc7 Pod IP \u76f4\u63a5\u8bbf\u95ee Overlay \u5b50\u7f51\u7684 Pod\u3002</p> <p>\u5982\u679c\u9700\u8981 Underlay \u548c Overlay \u4e92\u901a\u9700\u8981\u5c06\u5b50\u7f51\u7684 <code>u2oInterconnection</code> \u8bbe\u7f6e\u4e3a <code>true</code>\uff0c\u5728\u8fd9\u4e2a\u60c5\u51b5\u4e0b Kube-OVN \u4f1a\u989d\u5916\u4f7f\u7528\u4e00\u4e2a Underlay IP \u5c06 Underlay \u5b50\u7f51 \u548c <code>ovn-cluster</code> \u903b\u8f91\u8def\u7531\u5668\u8fde\u63a5\uff0c\u5e76\u8bbe\u7f6e\u5bf9\u5e94\u7684\u8def\u7531\u89c4\u5219\u5b9e\u73b0\u4e92\u901a\u3002 \u548c\u903b\u8f91\u7f51\u5173\u4e0d\u540c\uff0c\u8be5\u65b9\u6848\u53ea\u4f1a\u8fde\u63a5 Kube-OVN \u5185\u90e8\u7684 Underlay \u548c Overlay \u5b50\u7f51\uff0c\u5176\u4ed6\u8bbf\u95ee\u5916\u7f51\u7684\u6d41\u91cf\u8fd8\u662f\u4f1a\u901a\u8fc7\u7269\u7406\u7f51\u5173\u8fdb\u884c\u8f6c\u53d1\u3002</p>"},{"location":"start/underlay/#ip","title":"\u6307\u5b9a\u903b\u8f91\u7f51\u5173 IP","text":"<p>\u5f00\u542f\u4e92\u901a\u529f\u80fd\u540e\uff0c\u4f1a\u968f\u673a\u4ece subnet \u5185\u7684\u53d6\u4e00\u4e2a IP \u4f5c\u4e3a\u903b\u8f91\u7f51\u5173\uff0c\u5982\u679c\u9700\u8981\u6307\u5b9a Underlay Subnet \u7684\u903b\u8f91\u7f51\u5173\u53ef\u4ee5\u6307\u5b9a\u5b57\u6bb5 <code>u2oInterconnectionIP</code>\u3002</p>"},{"location":"start/underlay/#underlay-subnet-vpc","title":"\u6307\u5b9a Underlay Subnet \u8fde\u63a5\u7684\u81ea\u5b9a\u4e49 VPC","text":"<p>\u9ed8\u8ba4\u60c5\u51b5\u4e0b Underlay Subnet \u4f1a\u548c\u9ed8\u8ba4 VPC \u4e0a\u7684 Overlay Subnet \u4e92\u901a\uff0c\u5982\u679c\u8981\u6307\u5b9a\u548c\u67d0\u4e2a VPC \u4e92\u901a\uff0c\u5728 <code>u2oInterconnection</code> \u8bbe\u7f6e\u4e3a <code>true</code> \u540e\uff0c\u6307\u5b9a <code>subnet.spec.vpc</code> \u5b57\u6bb5\u4e3a\u8be5 VPC \u540d\u5b57\u5373\u53ef\u3002</p>"},{"location":"start/underlay/#_9","title":"\u6ce8\u610f\u4e8b\u9879","text":"<p>\u5982\u679c\u60a8\u4f7f\u7528\u7684\u8282\u70b9\u7f51\u5361\u4e0a\u914d\u7f6e\u6709 IP \u5730\u5740\uff0c\u4e14\u64cd\u4f5c\u7cfb\u7edf\u901a\u8fc7 Netplan \u914d\u7f6e\u7f51\u7edc\uff08\u5982 Ubuntu\uff09\uff0c\u5efa\u8bae\u60a8\u5c06 Netplan \u7684 renderer \u8bbe\u7f6e\u4e3a NetworkManager\uff0c\u5e76\u4e3a\u8282\u70b9\u7f51\u5361\u914d\u7f6e\u9759\u6001 IP \u5730\u5740\uff08\u5173\u95ed DHCP\uff09\uff1a</p> <pre><code>network:\nrenderer: NetworkManager\nethernets:\neth0:\ndhcp4: no\naddresses:\n- 172.16.143.129/24\nversion: 2\n</code></pre> <p>\u5982\u679c\u60a8\u8981\u4fee\u6539\u7f51\u5361\u7684 IP \u6216\u8def\u7531\u914d\u7f6e\uff0c\u9700\u8981\u5728\u4fee\u6539 netplan \u914d\u7f6e\u540e\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff1a</p> <pre><code>netplan generate\n\nnmcli connection reload netplan-eth0\nnmcli device set eth0 managed yes\n</code></pre> <p>\u6267\u884c\u4ee5\u4e0a\u547d\u4ee4\u540e\uff0cKube-OVN \u4f1a\u5c06\u7f51\u5361\u4e0a\u7684 IP \u53ca\u8def\u7531\u91cd\u65b0\u8f6c\u79fb\u81f3 OVS \u7f51\u6865\u3002</p> <p>\u5982\u679c\u60a8\u4f7f\u7528\u7684\u64cd\u4f5c\u7cfb\u7edf\u901a\u8fc7 NetworkManager \u7ba1\u7406\u7f51\u7edc\uff08\u5982 CentOS\uff09\uff0c\u5728\u4fee\u6539\u7f51\u5361\u914d\u7f6e\u540e\u9700\u8981\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff1a</p> <pre><code>nmcli connection reload eth0\nnmcli device set eth0 managed yes\nnmcli -t -f GENERAL.STATE device show eth0 | grep -qw unmanaged || nmcli device reapply eth0\n</code></pre> <p>\u6ce8\u610f\uff1a\u8282\u70b9\u7f51\u5361\u914d\u7f6e\u7684\u52a8\u6001\u4fee\u6539\u4ec5\u652f\u6301 IP \u548c\u8def\u7531\uff0c\u4e0d\u652f\u6301 MAC \u5730\u5740\u7684\u4fee\u6539\u3002</p>"},{"location":"start/underlay/#_10","title":"\u5df2\u77e5\u95ee\u9898","text":""},{"location":"start/underlay/#hairpin-pod","title":"\u7269\u7406\u7f51\u7edc\u5f00\u542f hairpin \u65f6 Pod \u7f51\u7edc\u5f02\u5e38","text":"<p>\u5f53\u7269\u7406\u7f51\u7edc\u5f00\u542f hairpin \u6216\u7c7b\u4f3c\u884c\u4e3a\u65f6\uff0c\u53ef\u80fd\u51fa\u73b0\u521b\u5efa Pod \u65f6\u7f51\u5173\u68c0\u67e5\u5931\u8d25\u3001Pod \u7f51\u7edc\u901a\u4fe1\u5f02\u5e38\u7b49\u95ee\u9898\u3002\u8fd9\u662f\u56e0\u4e3a OVS \u7f51\u6865\u9ed8\u8ba4\u7684 MAC \u5b66\u4e60\u529f\u80fd\u4e0d\u652f\u6301\u8fd9\u79cd\u7f51\u7edc\u73af\u5883\u3002</p> <p>\u8981\u89e3\u51b3\u6b64\u95ee\u9898\uff0c\u9700\u8981\u5173\u95ed hairpin\uff08\u6216\u4fee\u6539\u7269\u7406\u7f51\u7edc\u7684\u76f8\u5173\u914d\u7f6e\uff09\uff0c\u6216\u66f4\u65b0 Kube-OVN \u7248\u672c\u3002</p>"},{"location":"start/underlay/#pod-pod","title":"Pod \u6570\u91cf\u8f83\u591a\u65f6\u65b0\u5efa Pod \u7f51\u5173\u68c0\u67e5\u5931\u8d25","text":"<p>\u82e5\u540c\u4e00\u4e2a\u8282\u70b9\u4e0a\u8fd0\u884c\u7684 Pod \u6570\u91cf\u8f83\u591a\uff08\u5927\u4e8e 300\uff09\uff0c\u53ef\u80fd\u4f1a\u51fa\u73b0 ARP \u5e7f\u64ad\u5305\u7684 OVS \u6d41\u8868 resubmit \u6b21\u6570\u8d85\u8fc7\u4e0a\u9650\u5bfc\u81f4\u4e22\u5305\u7684\u73b0\u8c61\uff1a</p> <pre><code>2022-11-13T08:43:46.782Z|00222|ofproto_dpif_upcall(handler5)|WARN|Flow: arp,in_port=331,vlan_tci=0x0000,dl_src=00:00:00:25:eb:39,dl_dst=ff:ff:ff:ff:ff:ff,arp_spa=10.213.131.240,arp_tpa=10.213.159.254,arp_op=1,arp_sha=00:00:00:25:eb:39,arp_tha=ff:ff:ff:ff:ff:ff\n\nbridge(\"br-int\")\n----------------\n 0. No match.\n     &gt;&gt;&gt;&gt; received packet on unknown port 331 &lt;&lt;&lt;&lt;\n    drop\n\nFinal flow: unchanged\nMegaflow: recirc_id=0,eth,arp,in_port=331,dl_src=00:00:00:25:eb:39\nDatapath actions: drop\n2022-11-13T08:44:34.077Z|00224|ofproto_dpif_xlate(handler5)|WARN|over 4096 resubmit actions on bridge br-int while processing arp,in_port=13483,vlan_tci=0x0000,dl_src=00:00:00:59:ef:13,dl_dst=ff:ff:ff:ff:ff:ff,arp_spa=10.213.152.3,arp_tpa=10.213.159.254,arp_op=1,arp_sha=00:00:00:59:ef:13,arp_tha=ff:ff:ff:ff:ff:ff\n</code></pre> <p>\u8981\u89e3\u51b3\u6b64\u95ee\u9898\uff0c\u53ef\u4fee\u6539 OVN NB \u9009\u9879 <code>bcast_arp_req_flood</code> \u4e3a <code>false</code>\uff1a</p> <pre><code>kubectl ko nbctl set NB_Global . options:bcast_arp_req_flood=false\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"start/uninstall/","title":"\u5378\u8f7d","text":"<p>\u5982\u679c\u9700\u8981\u5220\u9664 Kube-OVN \u5e76\u66f4\u6362\u5176\u4ed6\u7f51\u7edc\u63d2\u4ef6\uff0c\u8bf7\u6309\u7167\u4e0b\u5217\u7684\u6b65\u9aa4\u5220\u9664\u5bf9\u5e94\u7684 Kube-OVN \u7ec4\u4ef6\u4ee5\u53ca OVS \u914d\u7f6e\uff0c\u4ee5\u907f\u514d\u5bf9\u5176\u4ed6\u7f51\u7edc\u63d2\u4ef6\u4ea7\u751f\u5e72\u6270\u3002 \u4e5f\u6b22\u8fce\u63d0 issue \u8054\u7cfb\u6211\u4eec\u53cd\u9988\u4e0d\u4f7f\u7528 Kube-OVN \u7684\u539f\u56e0\u5e2e\u52a9\u6211\u4eec\u6539\u8fdb\u3002</p>"},{"location":"start/uninstall/#kubernetes","title":"\u5220\u9664\u5728 Kubernetes \u4e2d\u521b\u5efa\u7684\u8d44\u6e90","text":"<p>\u4e0b\u8f7d\u4e0b\u9762\u7684\u811a\u672c\uff0c\u6267\u884c\u811a\u672c\u5220\u9664\u5728 Kubernetes \u4e2d\u521b\u5efa\u7684\u8d44\u6e90\uff1a</p> <pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/images/cleanup.sh\nbash cleanup.sh\n</code></pre>"},{"location":"start/uninstall/#_2","title":"\u6e05\u7406\u4e3b\u673a\u4e0a\u7684\u65e5\u5fd7\u548c\u914d\u7f6e\u6587\u4ef6","text":"<p>\u5728\u6bcf\u53f0\u673a\u5668\u4e0a\u6267\u884c\u4e0b\u5217\u64cd\u4f5c\uff0c\u6e05\u7406 ovsdb \u4ee5\u53ca openvswitch \u4fdd\u5b58\u7684\u914d\u7f6e\uff1a</p> <pre><code>rm -rf /var/run/openvswitch\nrm -rf /var/run/ovn\nrm -rf /etc/origin/openvswitch/\nrm -rf /etc/origin/ovn/\nrm -rf /etc/cni/net.d/00-kube-ovn.conflist\nrm -rf /etc/cni/net.d/01-kube-ovn.conflist\nrm -rf /var/log/openvswitch\nrm -rf /var/log/ovn\nrm -fr /var/log/kube-ovn\n</code></pre>"},{"location":"start/uninstall/#_3","title":"\u91cd\u542f\u8282\u70b9","text":"<p>\u91cd\u542f\u673a\u5668\u786e\u4fdd\u5bf9\u5e94\u7684\u7f51\u5361\u4fe1\u606f\uff0ciptable/ipset \u89c4\u5219\u5f97\u4ee5\u6e05\u9664\uff0c\u907f\u514d\u5bf9\u5176\u4ed6\u7f51\u7edc\u63d2\u4ef6\u7684\u5f71\u54cd\uff1a</p> <pre><code>reboot\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/","title":"Kube-OVN","text":"<p>Kube-OVN, a CNCF Sandbox Project, bridges the SDN into Cloud Native. It offers an advanced Container Network Fabric for Enterprises with the most functions, extreme performance and the easiest operation.</p> <p>Most Functions:</p> <p>If you miss the rich networking capabilities of the SDN age but are struggling to find them in the cloud-native age, Kube-OVN should be your best choice.</p> <p>Leveraging the proven capabilities of OVS/OVN in the SDN, Kube-OVN brings the rich capabilities of network virtualization to the cloud-native space. It currently supports Subnet Management, Static IP Allocation, Distributed/Centralized Gateways, Underlay/Overlay Hybrid Networks, VPC Multi-Tenant Networks, Cross-Cluster Interconnect, QoS Management, Multi-NIC Management, ACL, Traffic Mirroring, ARM Support, Windows Support, and many more.</p> <p>Extreme Performance:</p> <p>If you're concerned about the additional performance loss associated with container networks, then take a look at How Kube-OVN is doing everything it can to optimize performance.</p> <p>In the data plane, through a series of carefully optimized flow and kernel optimizations, and with emerging technologies such as eBPF, DPDK and SmartNIC Offload, Kube-OVN can approximate or exceed host network performance in terms of latency and throughput.</p> <p>In the control plane, Kube-OVN can support large-scale clusters of thousands of nodes and tens of thousands of Pods through the tailoring of OVN upstream flow tables and the use and tuning of various caching techniques.</p> <p>In addition, Kube-OVN is continuously optimizing the usage of resources such as CPU and memory to accommodate resource-limited scenarios such as the edge.</p> <p>Easiest Operation:</p> <p>If you're worried about container network operations, Kube-OVN has a number of built-in tools to help you simplify your operations.</p> <p>Kube-OVN provides one-click installation scripts to help users quickly build production-ready container networks. Also built-in rich monitoring metrics and Grafana dashboard help users to quickly set up monitoring system.</p> <p>Powerful command line tools simplify daily operations and maintenance for users. By combining with Cilium, users can enhance the observability of their networks with eBPF capabilities. In addition, the ability to mirror traffic makes it easy to customize traffic monitoring and interface with traditional NPM systems.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/contact/","title":"Contact US","text":"<p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/accelerate-intra-node-tcp-with-ebpf/","title":"Accelerate TCP Communication in Node with eBPF","text":"<p>At some edge and 5G scenarios, there will be a lot of TCP communication between Pods on the same node. By using the open source istio-tcpip-bypass project from Intel, Pods can use the ability of eBPF to bypass the host's TCP/IP protocol stack and communicate directly through sockets, thereby greatly reducing latency and improving throughput.</p>"},{"location":"en/advance/accelerate-intra-node-tcp-with-ebpf/#basic-principle","title":"Basic Principle","text":"<p>At present, two Pods on the same host need to go through a lot of network stacks, including TCP/IP, netfilter, OVS, etc., as shown in the following figure:</p> <p></p> <p>istio-tcpip-bypass plugin can automatically analyze and identify TCP communication within the same host, and bypass the complex kernel stack so that socket data transmission can be performed directly to reduce network stack processing overhead, as shown in the following figure:</p> <p></p> <p>Due to the fact that this component can automatically identify TCP communication within the same host and optimize it. In the Service Mesh environment based on the proxy mode, this component can also enhance the performance of Service Mesh.</p> <p>For more technical implementation details, please refer to Tanzu Service Mesh Acceleration using eBPF.</p>"},{"location":"en/advance/accelerate-intra-node-tcp-with-ebpf/#prerequisites","title":"Prerequisites","text":"<p>eBPF requires a kernel version of at least 5.4.0-74-generic. It is recommended to use Ubuntu 20.04 and Linux 5.4.0-74-generic kernel version for testing.</p>"},{"location":"en/advance/accelerate-intra-node-tcp-with-ebpf/#experimental-steps","title":"Experimental Steps","text":"<p>Deploy two performance test Pods on the same node. If there are multiple machines in the cluster, you need to specify <code>nodeSelector</code>:</p> <pre><code># kubectl create deployment perf --image=kubeovn/perf:dev --replicas=2\ndeployment.apps/perf created\n# kubectl get pod -o wide\nNAME                    READY   STATUS    RESTARTS   AGE   IP           NODE     NOMINATED NODE   READINESS GATES\nperf-7697bc6ddf-b2cpv   1/1     Running   0          28s   100.64.0.3   sealos   &lt;none&gt;           &lt;none&gt;\nperf-7697bc6ddf-p2xpt   1/1     Running   0          28s   100.64.0.2   sealos   &lt;none&gt;           &lt;none&gt;\n</code></pre> <p>Enter one of the Pods to start the qperf server, and start the qperf client in another Pod for performance testing:</p> <pre><code># kubectl exec -it perf-7697bc6ddf-b2cpv sh\n/ # qperf\n\n# kubectl exec -it perf-7697bc6ddf-p2xpt sh\n/ # qperf -t 60 100.64.0.3 -ub -oo msg_size:1:16K:*4 -vu tcp_lat tcp_bw\n</code></pre> <p>Deploy the istio-tcpip-bypass plugin:</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/intel/istio-tcpip-bypass/main/bypass-tcpip-daemonset.yaml\n</code></pre> <p>Enter the perf client container again for performance testing:</p> <pre><code># kubectl exec -it perf-7697bc6ddf-p2xpt sh\n/ # qperf -t 60 100.64.0.3 -ub -oo msg_size:1:16K:*4 -vu tcp_lat tcp_bw\n</code></pre>"},{"location":"en/advance/accelerate-intra-node-tcp-with-ebpf/#test-results","title":"Test Results","text":"<p>According to the test results, the TCP latency will decrease by 40% ~ 60% under different packet sizes, and the throughput will increase by 40% ~ 80% when the packet size is greater than 1024 bytes.</p> Packet Size (byte) eBPF tcp_lat (us) Default tcp_lat (us) eBPF tcp_bw (Mb/s) Default tcp_bw(Mb/s) 1 20.2 44.5 1.36 4.27 4 20.2 48.7 5.48 16.7 16 19.6 41.6 21.7 63.5 64 18.8 41.3 96.8 201 256 19.2 36 395 539 1024 18.3 42.4 1360 846 4096 16.5 62.6 4460 2430 16384 20.2 58.8 9600 6900 <p>In the hardware environment under test, when the packet size is less than 512 bytes, the throughput indicator optimized by eBPF is lower than the throughput under the default configuration. This situation may be related to the TCP aggregation optimization of the network card under the default configuration. If the application scenario is sensitive to small packet throughput, you need to test in the corresponding environment Determine whether to enable eBPF optimization. We will also optimize the throughput of eBPF TCP small packet scenarios in the future.</p>"},{"location":"en/advance/accelerate-intra-node-tcp-with-ebpf/#references","title":"References","text":"<ol> <li>istio-tcpip-bypass</li> <li>Deep Dive TCP/IP Bypass with eBPF in Service Mesh</li> <li>Tanzu Service Mesh Acceleration using eBPF</li> </ol> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/cilium-hubble-observe/","title":"Cilium Network Traffic Observation","text":"<p>Kube-OVN supports Cilium integration, please refer to Cilium integration for details.</p> <p>Cilium provides rich network traffic observation capabilities, and the flow observability is provided by Hubble. Hubble can observe the traffic across nodes, clusters, and even multi-cluster scenarios.</p>"},{"location":"en/advance/cilium-hubble-observe/#install-hubble","title":"Install Hubble","text":"<p>In the default Cilium integration installation, the Hubble related components are not installed, so to support traffic observation, you need to supplement the installation of Hubble on the environment.</p> <p>Execute the following command to install Hubble using helm:</p> <pre><code>helm upgrade cilium cilium/cilium --version 1.11.6 \\\n--namespace kube-system \\\n--reuse-values \\\n--set hubble.relay.enabled=true \\\n--set hubble.ui.enabled=true\n</code></pre> <p>After installing Hubble, execute <code>cilium status</code> to check the status of the component and confirm that the installation is successful.</p> <pre><code># cilium status\n/\u00af\u00af\\\n/\u00af\u00af\\__/\u00af\u00af\\    Cilium:         OK\n \\__/\u00af\u00af\\__/    Operator:       OK\n /\u00af\u00af\\__/\u00af\u00af\\    Hubble:         OK\n \\__/\u00af\u00af\\__/    ClusterMesh:    disabled\n    \\__/\n\nDeployment        hubble-relay       Desired: 1, Ready: 1/1, Available: 1/1\nDeployment        cilium-operator    Desired: 2, Ready: 2/2, Available: 2/2\nDaemonSet         cilium             Desired: 2, Ready: 2/2, Available: 2/2\nDeployment        hubble-ui          Desired: 1, Ready: 1/1, Available: 1/1\nContainers:       cilium             Running: 2\nhubble-ui          Running: 1\nhubble-relay       Running: 1\ncilium-operator    Running: 2\nCluster Pods:     16/17 managed by Cilium\nImage versions    hubble-relay       quay.io/cilium/hubble-relay:v1.11.6@sha256:fd9034a2d04d5b973f1e8ed44f230ea195b89c37955ff32e34e5aa68f3ed675a: 1\ncilium-operator    quay.io/cilium/operator-generic:v1.11.6@sha256:9f6063c7bcaede801a39315ec7c166309f6a6783e98665f6693939cf1701bc17: 2\ncilium             quay.io/cilium/cilium:v1.11.6@sha256:f7f93c26739b6641a3fa3d76b1e1605b15989f25d06625260099e01c8243f54c: 2\nhubble-ui          quay.io/cilium/hubble-ui:v0.9.0@sha256:0ef04e9a29212925da6bdfd0ba5b581765e41a01f1cc30563cef9b30b457fea0: 1\nhubble-ui          quay.io/cilium/hubble-ui-backend:v0.9.0@sha256:000df6b76719f607a9edefb9af94dfd1811a6f1b6a8a9c537cba90bf12df474b: 1\napple@bogon cilium %\n</code></pre> <p>After installing the Hubble component, you need to install the command line to view the traffic information in the environment. Execute the following command to install Hubble CLI:</p> <pre><code>curl -L --fail --remote-name-all https://github.com/cilium/hubble/releases/download/v0.10.0/hubble-linux-amd64.tar.gz\nsudo tar xzvfC hubble-linux-amd64.tar.gz /usr/local/bin\n</code></pre>"},{"location":"en/advance/cilium-hubble-observe/#deploy-and-test","title":"Deploy and test","text":"<p>Cilium offers a traffic test deployment solution, you can directly use the official deployment solution to deploy the test.</p> <p>Execute the command <code>cilium connectivity test</code>, Cilium will automatically create the <code>cilium-test</code> namespace, and deploy the test under cilium-test.</p> <p>After the normal deployment, you can view the resource information under the <code>cilium-test</code> namespace, as follows:</p> <pre><code># kubectl get all -n cilium-test\nNAME                                  READY   STATUS    RESTARTS   AGE\npod/client-7df6cfbf7b-z5t2j           1/1     Running   0          21s\npod/client2-547996d7d8-nvgxg          1/1     Running   0          21s\npod/echo-other-node-d79544ccf-hl4gg   2/2     Running   0          21s\npod/echo-same-node-5d466d5444-ml7tc   2/2     Running   0          21s\n\nNAME                      TYPE       CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE\nservice/echo-other-node   NodePort   10.109.58.126   &lt;none&gt;        8080:32269/TCP   21s\nservice/echo-same-node    NodePort   10.108.70.32    &lt;none&gt;        8080:32490/TCP   21s\n\nNAME                              READY   UP-TO-DATE   AVAILABLE   AGE\ndeployment.apps/client            1/1     1            1           21s\ndeployment.apps/client2           1/1     1            1           21s\ndeployment.apps/echo-other-node   1/1     1            1           21s\ndeployment.apps/echo-same-node    1/1     1            1           21s\n\nNAME                                        DESIRED   CURRENT   READY   AGE\nreplicaset.apps/client-7df6cfbf7b           1         1         1       21s\nreplicaset.apps/client2-547996d7d8          1         1         1       21s\nreplicaset.apps/echo-other-node-d79544ccf   1         1         1       21s\nreplicaset.apps/echo-same-node-5d466d5444   1         1         1       21s\n</code></pre>"},{"location":"en/advance/cilium-hubble-observe/#use-the-command-line-to-observe-traffic","title":"Use the command line to observe traffic","text":"<p>By default, the network traffic observation only provides the traffic observed by the Cilium agent on each node.</p> <p>Execute the <code>hubble observe</code> command in the Cilium agent pod under the <code>kube-system namespace</code> to view the traffic information on the node.</p> <pre><code># kubectl get pod -n kube-system -o wide\nNAME                                             READY   STATUS    RESTARTS   AGE     IP           NODE                     NOMINATED NODE   READINESS GATES\ncilium-d6h56                                     1/1     Running   0          2d20h   172.18.0.2   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;\ncilium-operator-5887f78bbb-c7sb2                 1/1     Running   0          2d20h   172.18.0.2   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;\ncilium-operator-5887f78bbb-wj8gt                 1/1     Running   0          2d20h   172.18.0.3   kube-ovn-control-plane   &lt;none&gt;           &lt;none&gt;\ncilium-tq5xb                                     1/1     Running   0          2d20h   172.18.0.3   kube-ovn-control-plane   &lt;none&gt;           &lt;none&gt;\nkube-ovn-pinger-7lgk8                            1/1     Running   0          21h     10.16.0.19   kube-ovn-control-plane   &lt;none&gt;           &lt;none&gt;\nkube-ovn-pinger-msvcn                            1/1     Running   0          21h     10.16.0.18   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;\n\n# kubectl exec -it -n kube-system cilium-d6h56 -- bash\nroot@kube-ovn-worker:/home/cilium# hubble observe --from-namespace kube-system\nJul 29 03:24:25.551: kube-system/kube-ovn-pinger-msvcn:35576 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: ACK, PSH)\nJul 29 03:24:25.561: kube-system/kube-ovn-pinger-msvcn:35576 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: RST)\nJul 29 03:24:25.561: kube-system/kube-ovn-pinger-msvcn:35576 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: ACK, RST)\nJul 29 03:24:25.572: kube-system/kube-ovn-pinger-msvcn:35578 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: SYN)\nJul 29 03:24:25.572: kube-system/kube-ovn-pinger-msvcn:35578 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: ACK)\nJul 29 03:24:25.651: kube-system/kube-ovn-pinger-msvcn:35578 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: ACK, PSH)\nJul 29 03:24:25.661: kube-system/kube-ovn-pinger-msvcn:35578 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: RST)\nJul 29 03:24:25.661: kube-system/kube-ovn-pinger-msvcn:35578 -&gt; 172.18.0.3:6642 to-stack FORWARDED (TCP Flags: ACK, RST)\nJul 29 03:24:25.761: kube-system/kube-ovn-pinger-msvcn:52004 -&gt; 172.18.0.3:6443 to-stack FORWARDED (TCP Flags: ACK, PSH)\nJul 29 03:24:25.779: kube-system/kube-ovn-pinger-msvcn -&gt; kube-system/kube-ovn-pinger-7lgk8 to-stack FORWARDED (ICMPv4 EchoRequest)\nJul 29 03:24:25.779: kube-system/kube-ovn-pinger-msvcn &lt;- kube-system/kube-ovn-pinger-7lgk8 to-endpoint FORWARDED (ICMPv4 EchoReply)\nJul 29 03:24:25.866: kube-system/hubble-ui-7596f7ff6f-7j6f2:55836 &lt;- kube-system/hubble-relay-959988db5-zc5vv:4245 to-stack FORWARDED (TCP Flags: ACK)\nJul 29 03:24:25.866: kube-system/hubble-ui-7596f7ff6f-7j6f2:55836 &lt;- kube-system/hubble-relay-959988db5-zc5vv:80 to-endpoint FORWARDED (TCP Flags: ACK)\nJul 29 03:24:25.866: kube-system/hubble-ui-7596f7ff6f-7j6f2:55836 -&gt; kube-system/hubble-relay-959988db5-zc5vv:4245 to-stack FORWARDED (TCP Flags: ACK)\nJul 29 03:24:25.866: kube-system/hubble-ui-7596f7ff6f-7j6f2:55836 -&gt; kube-system/hubble-relay-959988db5-zc5vv:4245 to-endpoint FORWARDED (TCP Flags: ACK)\nJul 29 03:24:25.975: kube-system/kube-ovn-pinger-7lgk8 -&gt; kube-system/kube-ovn-pinger-msvcn to-endpoint FORWARDED (ICMPv4 EchoRequest)\nJul 29 03:24:25.975: kube-system/kube-ovn-pinger-7lgk8 &lt;- kube-system/kube-ovn-pinger-msvcn to-stack FORWARDED (ICMPv4 EchoReply)\nJul 29 03:24:25.979: kube-system/kube-ovn-pinger-msvcn -&gt; 172.18.0.3 to-stack FORWARDED (ICMPv4 EchoRequest)\nJul 29 03:24:26.037: kube-system/coredns-6d4b75cb6d-lbgjg:36430 -&gt; 172.18.0.3:6443 to-stack FORWARDED (TCP Flags: ACK)\nJul 29 03:24:26.282: kube-system/kube-ovn-pinger-msvcn -&gt; 172.18.0.2 to-stack FORWARDED (ICMPv4 EchoRequest)\n</code></pre> <p>After deploying Hubble Relay, Hubble can provide complete cluster-wide network traffic observation.</p>"},{"location":"en/advance/cilium-hubble-observe/#configure-port-forwarding","title":"Configure port forwarding","text":"<p>In order to access the Hubble API normally, you need to create a port forwarding to forward the local request to the Hubble Service. You can execute the <code>kubectl port-forward deployment/hubble-relay -n kube-system 4245:4245</code> command to open the port forwarding in the current terminal.</p> <p>The port forwarding configuration can refer to Port Forwarding.</p> <p><code>kubectl port-forward</code> is a blocking command, you can open a new terminal to execute the following command to observe the traffic information.</p> <p>After configuring the port forwarding, execute the <code>hubble status</code> command in the terminal. If there is an output similar to the following, the port forwarding configuration is correct, and you can use the command line to observe the traffic.</p> <pre><code># hubble status\nHealthcheck (via localhost:4245): Ok\nCurrent/Max Flows: 8,190/8,190 (100.00%)\nFlows/s: 22.86\nConnected Nodes: 2/2\n</code></pre>"},{"location":"en/advance/cilium-hubble-observe/#use-the-command-line-to-observe-traffic_1","title":"Use the command line to observe traffic","text":"<p>Execute the <code>hubble observe</code> command in the terminal to view the traffic information of the cluster.</p> <p>The traffic observed by the <code>cilium-test</code> namespace is as follows:</p> <p></p> <p>Pay attention to the <code>hubble observe</code> command display result, which is the traffic information queried when the current command line is executed. Executing the command line multiple times can view different traffic information. For more detailed observation information, you can execute the <code>hubble help observe</code> command to view the detailed usage of Hubble CLI.</p>"},{"location":"en/advance/cilium-hubble-observe/#use-ui-to-observe-traffic","title":"Use UI to observe traffic","text":"<p>Execute the <code>cilium status</code> command to confirm that the Hubble UI has been successfully installed. In the second step of the Hubble installation, the installation of the UI has been supplemented.</p> <p>Execute the command <code>cilium hubble ui</code> to automatically create port forwarding and map the <code>hubble-ui service</code> to the local port.</p> <p>When the command is executed normally, the local browser will be automatically opened and jump to the Hubble UI interface. If it does not jump automatically, enter <code>http://localhost:12000</code> in the browser to open the UI observation interface.</p> <p>On the top left of the UI, select the <code>cilium-test</code> namespace to view the test traffic information provided by Cilium. </p>"},{"location":"en/advance/cilium-hubble-observe/#hubble-traffic-monitoring","title":"Hubble Traffic Monitoring","text":"<p>Hubble component provides monitoring of Pod network behavior in the cluster. In order to support viewing the monitoring data provided by Hubble, you need to enable monitoring statistics.</p> <p>Refer to the following command to supplement the <code>hubble.metrics.enabled</code> configuration item:</p> <pre><code>helm upgrade cilium cilium/cilium --version 1.11.6 \\\n--namespace kube-system \\\n--reuse-values \\\n--set hubble.relay.enabled=true \\\n--set hubble.ui.enabled=true \\\n--set hubble.metrics.enabled=\"{dns,drop,tcp,flow,icmp,http}\"\n</code></pre> <p>After the deployment is completed, you can view the monitoring data provided by Hubble through the <code>hubble-metrics</code> service. Execute the following command to view the monitoring data:</p> <pre><code># curl 172.18.0.2:9091/metrics\n# HELP hubble_drop_total Number of drops\n# TYPE hubble_drop_total counter\nhubble_drop_total{protocol=\"ICMPv6\",reason=\"Unsupported L3 protocol\"} 2\n# HELP hubble_flows_processed_total Total number of flows processed\n# TYPE hubble_flows_processed_total counter\nhubble_flows_processed_total{protocol=\"ICMPv4\",subtype=\"to-endpoint\",type=\"Trace\",verdict=\"FORWARDED\"} 335\nhubble_flows_processed_total{protocol=\"ICMPv4\",subtype=\"to-stack\",type=\"Trace\",verdict=\"FORWARDED\"} 335\nhubble_flows_processed_total{protocol=\"ICMPv6\",subtype=\"\",type=\"Drop\",verdict=\"DROPPED\"} 2\nhubble_flows_processed_total{protocol=\"TCP\",subtype=\"to-endpoint\",type=\"Trace\",verdict=\"FORWARDED\"} 8282\nhubble_flows_processed_total{protocol=\"TCP\",subtype=\"to-stack\",type=\"Trace\",verdict=\"FORWARDED\"} 6767\nhubble_flows_processed_total{protocol=\"UDP\",subtype=\"to-endpoint\",type=\"Trace\",verdict=\"FORWARDED\"} 1642\nhubble_flows_processed_total{protocol=\"UDP\",subtype=\"to-stack\",type=\"Trace\",verdict=\"FORWARDED\"} 1642\n# HELP hubble_icmp_total Number of ICMP messages\n# TYPE hubble_icmp_total counter\nhubble_icmp_total{family=\"IPv4\",type=\"EchoReply\"} 335\nhubble_icmp_total{family=\"IPv4\",type=\"EchoRequest\"} 335\nhubble_icmp_total{family=\"IPv4\",type=\"RouterSolicitation\"} 2\n# HELP hubble_tcp_flags_total TCP flag occurrences\n# TYPE hubble_tcp_flags_total counter\nhubble_tcp_flags_total{family=\"IPv4\",flag=\"FIN\"} 2043\nhubble_tcp_flags_total{family=\"IPv4\",flag=\"RST\"} 301\nhubble_tcp_flags_total{family=\"IPv4\",flag=\"SYN\"} 1169\nhubble_tcp_flags_total{family=\"IPv4\",flag=\"SYN-ACK\"} 1169\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/cilium-networkpolicy/","title":"Cilium NetworkPolicy Support","text":"<p>Kube-OVN currently supports integration with Cilium, and the specific operation can refer to Cilium integration.</p> <p>After integrating Cilium, you can use Cilium's excellent network policy capabilities to control the access of Pods in the cluster.The following documents provide integration verification of Cilium L3 and L4 network policy capabilities.</p>"},{"location":"en/advance/cilium-networkpolicy/#verification-steps","title":"Verification Steps","text":""},{"location":"en/advance/cilium-networkpolicy/#create-test-pod","title":"Create test Pod","text":"<p>Create namespace <code>test</code>. Refer to the following yaml, create Pod with label <code>app=test</code> in namespace <code>test</code> as the destination Pod for testing access.</p> <pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nlabels:\napp: test\nname: test\nnamespace: test\nspec:\nreplicas: 1\nselector:\nmatchLabels:\napp: test\nstrategy:\nrollingUpdate:\nmaxSurge: 25%\nmaxUnavailable: 25%\ntype: RollingUpdate\ntemplate:\nmetadata:\nlabels:\napp: test\nspec:\ncontainers:\n- image: docker.io/library/nginx:alpine\nimagePullPolicy: IfNotPresent\nname: nginx\n</code></pre> <p>Similarly, refer to the following yaml, create Pod with label <code>app=dynamic</code> in namespace <code>default</code> as the Pod for testing access.</p> <pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nlabels:\napp: dynamic\nname: dynamic\nnamespace: default\nspec:\nreplicas: 2\nselector:\nmatchLabels:\napp: dynamic\nstrategy:\nrollingUpdate:\nmaxSurge: 25%\nmaxUnavailable: 25%\ntype: RollingUpdate\ntemplate:\nmetadata:\ncreationTimestamp: null\nlabels:\napp: dynamic\nspec:\ncontainers:\n- image: docker.io/library/nginx:alpine\nimagePullPolicy: IfNotPresent\nname: nginx\n</code></pre> <p>View the test Pod and Label information:</p> <pre><code># kubectl get pod -o wide --show-labels\nNAME                         READY   STATUS    RESTARTS   AGE   IP           NODE                     NOMINATED NODE   READINESS GATES   LABELS\ndynamic-7d8d7874f5-9v5c4     1/1     Running   0          28h   10.16.0.35   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;            app=dynamic,pod-template-hash=7d8d7874f5\ndynamic-7d8d7874f5-s8z2n     1/1     Running   0          28h   10.16.0.36   kube-ovn-control-plane   &lt;none&gt;           &lt;none&gt;            app=dynamic,pod-template-hash=7d8d7874f5\n# kubectl get pod -o wide -n test --show-labels\nNAME                           READY   STATUS    RESTARTS   AGE     IP           NODE                     NOMINATED NODE   READINESS GATES   LABELS\ndynamic-7d8d7874f5-6dsg6       1/1     Running   0          7h20m   10.16.0.2    kube-ovn-control-plane   &lt;none&gt;           &lt;none&gt;            app=dynamic,pod-template-hash=7d8d7874f5\ndynamic-7d8d7874f5-tjgtp       1/1     Running   0          7h46m   10.16.0.42   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;            app=dynamic,pod-template-hash=7d8d7874f5\nlabel-test1-77b6764857-swq4k   1/1     Running   0          3h43m   10.16.0.12   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;            app=test1,pod-template-hash=77b6764857\n\n// As the destination Pod for testing access.\ntest-54c98bc466-mft5s          1/1     Running   0          8h      10.16.0.41   kube-ovn-worker          &lt;none&gt;           &lt;none&gt;            app=test,pod-template-hash=54c98bc466\n</code></pre>"},{"location":"en/advance/cilium-networkpolicy/#l3-network-policy-test","title":"L3 Network Policy Test","text":"<p>Refer to the following yaml, create <code>CiliumNetworkPolicy</code> resource:</p> <pre><code>apiVersion: \"cilium.io/v2\"\nkind: CiliumNetworkPolicy\nmetadata:\nname: \"l3-rule\"\nnamespace: test\nspec:\nendpointSelector:\nmatchLabels:\napp: test\ningress:\n- fromEndpoints:\n- matchLabels:\napp: dynamic\n</code></pre> <p>At this point, the test Pod in the default namespace cannot access the destination Pod, but the test Pod to the destination Pod in the test namespace is accessible.</p> <p>Test results in the default namespace:</p> <pre><code># kubectl exec -it dynamic-7d8d7874f5-9v5c4 -- bash\nbash-5.0# ping -c 3 10.16.0.41\nPING 10.16.0.41 (10.16.0.41): 56 data bytes\n\n--- 10.16.0.41 ping statistics ---\n3 packets transmitted, 0 packets received, 100% packet loss\n</code></pre> <p>Test results in the test namespace:</p> <pre><code># kubectl exec -it -n test dynamic-7d8d7874f5-6dsg6 -- bash\nbash-5.0# ping -c 3 10.16.0.41\nPING 10.16.0.41 (10.16.0.41): 56 data bytes\n64 bytes from 10.16.0.41: seq=0 ttl=64 time=2.558 ms\n64 bytes from 10.16.0.41: seq=1 ttl=64 time=0.223 ms\n64 bytes from 10.16.0.41: seq=2 ttl=64 time=0.304 ms\n\n--- 10.16.0.41 ping statistics ---\n3 packets transmitted, 3 packets received, 0% packet loss\nround-trip min/avg/max = 0.223/1.028/2.558 ms\n</code></pre> <p>Look at the Cilium official document explanation, the <code>CiliumNetworkPolicy</code> resource limits the control at the <code>namespace</code> level. For more information, please refer to Cilium Limitations.</p> <p>If there is a network policy rule match, only the Pod in the same namespace can access according to the rule, and the Pod in the other namespace is denied access by default.</p> <p>If you want to implement cross-namespace access, you need to specify the namespace information in the rule.</p> <p>Refer to the document, modify the <code>CiliumNetworkPolicy</code> resource, and add namespace information:</p> <pre><code>  ingress:\n- fromEndpoints:\n- matchLabels:\napp: dynamic\nk8s:io.kubernetes.pod.namespace: default    // control the Pod access in other namespace\n</code></pre> <p>Look at the modified <code>CiliumNetworkPolicy</code> resource information:</p> <pre><code># kubectl get cnp -n test  -o yaml l3-rule\napiVersion: cilium.io/v2\nkind: CiliumNetworkPolicy\nmetadata:\n  name: l3-rule\n  namespace: test\nspec:\n  endpointSelector:\n    matchLabels:\n      app: test\ningress:\n  - fromEndpoints:\n    - matchLabels:\n        app: dynamic\n    - matchLabels:\n        app: dynamic\n        k8s:io.kubernetes.pod.namespace: default\n</code></pre> <p>Test the Pod access in the default namespace again, and the destination Pod access is normal:</p> <pre><code># kubectl exec -it dynamic-7d8d7874f5-9v5c4 -n test -- bash\nbash-5.0# ping -c 3 10.16.0.41\nPING 10.16.0.41 (10.16.0.41): 56 data bytes\n64 bytes from 10.16.0.41: seq=0 ttl=64 time=2.383 ms\n64 bytes from 10.16.0.41: seq=1 ttl=64 time=0.115 ms\n64 bytes from 10.16.0.41: seq=2 ttl=64 time=0.142 ms\n\n--- 10.16.0.41 ping statistics ---\n3 packets transmitted, 3 packets received, 0% packet loss\nround-trip min/avg/max = 0.115/0.880/2.383 ms\n</code></pre> <p>Using the standard Kubernetes network policy networkpolicy, the test results show that Cilium also restricts access within the same namespace, and cross-namespace access is prohibited.</p> <p>It is different from Kube-OVN implementation. Kube-OVN supports standard k8s network policy, which restricts the destination Pod in a specific namespace, but there is no namespace restriction on the source Pod. Any Pod that meets the restriction rules in any namespace can access the destination Pod.</p>"},{"location":"en/advance/cilium-networkpolicy/#l4-network-policy-test","title":"L4 Network Policy Test","text":"<p>Refer to the following yaml, create <code>CiliumNetworkPolicy</code> resource:</p> <pre><code>apiVersion: \"cilium.io/v2\"\nkind: CiliumNetworkPolicy\nmetadata:\nname: \"l4-rule\"\nnamespace: test\nspec:\nendpointSelector:\nmatchLabels:\napp: test\ningress:\n- fromEndpoints:\n- matchLabels:\napp: dynamic\ntoPorts:\n- ports:\n- port: \"80\"\nprotocol: TCP\n</code></pre> <p>Test the access of the Pod that meets the network policy rules in the same namespace</p> <pre><code># kubectl exec -it -n test dynamic-7d8d7874f5-6dsg6 -- bash\nbash-5.0# ping -c 3 10.16.0.41\nPING 10.16.0.41 (10.16.0.41): 56 data bytes\n\n--- 10.16.0.41 ping statistics ---\n3 packets transmitted, 0 packets received, 100% packet loss\nbash-5.0#\nbash-5.0# curl 10.16.0.41:80\n&lt;html&gt;\n&lt;head&gt;\n        &lt;title&gt;Hello World!&lt;/title&gt;\n        &lt;link href='//fonts.googleapis.com/css?family=Open+Sans:400,700' rel='stylesheet' type='text/css'&gt;\n        &lt;style&gt;\n        body {\nbackground-color: white;\ntext-align: center;\npadding: 50px;\nfont-family: \"Open Sans\",\"Helvetica Neue\",Helvetica,Arial,sans-serif;\n}\n#logo {\nmargin-bottom: 40px;\n}\n&lt;/style&gt;\n&lt;/head&gt;\n&lt;body&gt;\n                &lt;h1&gt;Hello World!&lt;/h1&gt;\n                                &lt;h3&gt;Links found&lt;/h3&gt;\n        &lt;h3&gt;I am on  test-54c98bc466-mft5s&lt;/h3&gt;\n        &lt;h3&gt;Cookie                  =&lt;/h3&gt;\n                                        &lt;b&gt;KUBERNETES&lt;/b&gt; listening in 443 available at tcp://10.96.0.1:443&lt;br /&gt;\n                                                &lt;h3&gt;my name is hanhouchao!&lt;/h3&gt;\n                        &lt;h3&gt; RequestURI='/'&lt;/h3&gt;\n&lt;/body&gt;\n&lt;/html&gt;\n</code></pre> <p>The Pod that does not meet the network policy rules in the same namespace cannot access</p> <pre><code># kubectl exec -it -n test label-test1-77b6764857-swq4k -- bash\nbash-5.0# ping -c 3 10.16.0.41\nPING 10.16.0.41 (10.16.0.41): 56 data bytes\n\n--- 10.16.0.41 ping statistics ---\n3 packets transmitted, 0 packets received, 100% packet loss\nbash-5.0#\nbash-5.0# curl -v 10.16.0.41:80 --connect-timeout 10\n*   Trying 10.16.0.41:80...\n* After 10000ms connect time, move on!\n* connect to 10.16.0.41 port 80 failed: Operation timed out\n* Connection timeout after 10001 ms\n* Closing connection 0\ncurl: (28) Connection timeout after 10001 ms\n</code></pre> <p>After the network policy takes effect, cross-namespace access is still prohibited, which is consistent with the L3 network policy test results.</p> <p>After the L4 network policy takes effect, ping cannot be used, but TCP access that meets the policy rules can be executed normally.</p> <p>About the restriction of ICMP, please refer to the official description L4 Limitation Description.</p>"},{"location":"en/advance/cilium-networkpolicy/#l7-network-policy-test","title":"L7 Network Policy Test","text":"<p>chaining mode, L7 network policy currently has problems. In the Cilium official document, there is an explanation for this situation, please refer to Generic Veth Chaining.</p> <p>This problem is tracked using issue 12454, and it has not been resolved yet.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/dhcp/","title":"DHCP","text":"<p>When using SR-IOV or DPDK type networks, KubeVirt's built-in DHCP does not work in this network mode. Kube-OVN can use the DHCP capabilities of OVN to set DHCP options at the subnet level to help KubeVirt VMs of these network types to properly use DHCP to obtain assigned IP addresses. Kube-OVN supports both DHCPv4 and DHCPv6.</p> <p>The subnet DHCP is configured as follows:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: sn-dual\nspec:\ncidrBlock: \"10.0.0.0/24,240e::a00/120\"\ndefault: false\ndisableGatewayCheck: true\ndisableInterConnection: false\nexcludeIps:\n- 10.0.0.1\n- 240e::a01\ngateway: 10.0.0.1,240e::a01\ngatewayNode: ''\ngatewayType: distributed\nnatOutgoing: false\nprivate: false\nprotocol: Dual\nprovider: ovn\nvpc: vpc-test\nenableDHCP: true\ndhcpV4Options: \"lease_time=3600,router=10.0.0.1,server_id=169.254.0.254,server_mac=00:00:00:2E:2F:B8\"\ndhcpV6Options: \"server_id=00:00:00:2E:2F:C5\"\nenableIPv6RA: true\nipv6RAConfigs: \"address_mode=dhcpv6_stateful,max_interval=30,min_interval=5,send_periodic=true\"\n</code></pre> <ul> <li><code>enableDHCP</code>: Whether to enable the DHCP function for the subnet.</li> <li><code>dhcpV4Options</code>,<code>dhcpV6Options</code>: This field directly exposes DHCP-related options within ovn-nb, please reade DHCP Options for more detail. The default value is  <code>\"lease_time=3600, router=$ipv4_gateway, server_id=169.254.0.254, server_mac=$random_mac\"</code> and <code>server_id=$random_mac</code>\u3002</li> <li><code>enableIPv6RA</code>: Whether to enable the route broadcast function of DHCPv6.</li> <li><code>ipv6RAConfigs</code>\uff1aThis field directly exposes DHCP-related options within ovn-nb Logical_Router_Port, please read Logical Router Port for more detail. The default value is <code>address_mode=dhcpv6_stateful, max_interval=30, min_interval=5, send_periodic=true</code>\u3002</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/dpdk/","title":"DPDK Support","text":"<p>This document describes how Kube-OVN combines with OVS-DPDK to provide a DPDK-type network interface to KubeVirt's virtual machines.</p> <p>Upstream KubeVirt does not currently support OVS-DPDK, users need to use the downstream patch Vhostuser implementation to build KubeVirt by themselves or KVM Device Plugin to use OVS-DPDK.</p>"},{"location":"en/advance/dpdk/#prerequisites","title":"Prerequisites","text":"<ul> <li>The node needs to provide a dedicated NIC for the DPDK driver to run.</li> <li>The node needs to have Hugepages enabled.</li> </ul>"},{"location":"en/advance/dpdk/#set-dpdk-driver","title":"Set DPDK driver","text":"<p>Here we use <code>driverctl</code> for example, please refer to the DPDK documentation for specific parameters and other driver usage:</p> <pre><code>driverctl set-override 0000:00:0b.0 uio_pci_generic\n</code></pre>"},{"location":"en/advance/dpdk/#configure-nodes","title":"Configure Nodes","text":"<p>Labeling OVS-DPDK-enabled nodes for Kube-OVN to recognize:</p> <pre><code>kubectl label nodes &lt;node&gt; ovn.kubernetes.io/ovs_dp_type=\"userspace\"\n</code></pre> <p>Create the configuration file <code>ovs-dpdk-config</code> in the <code>/opt/ovs-config</code> directory on nodes that support DPDK.</p> <pre><code>ENCAP_IP=192.168.122.193/24\nDPDK_DEV=0000:00:0b.0\n</code></pre> <ul> <li><code>ENCAP_IP</code>: The tunnel endpoint address.</li> <li><code>DPDK_DEV</code>: The PCI ID of the device.</li> </ul>"},{"location":"en/advance/dpdk/#install-kube-ovn","title":"Install Kube-OVN","text":"<p>Download scripts:</p> <pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/images/install.sh\n</code></pre> <p>Enable the DPDK installation option:</p> <pre><code>bash install.sh --with-hybrid-dpdk\n</code></pre>"},{"location":"en/advance/dpdk/#usage","title":"Usage","text":"<p>Here we verify the OVS-DPDK functionality by creating a virtual machine with a vhostuser type NIC.</p> <p>Here we use the KVM Device Plugin to create virtual machines. For more information on how to use it, please refer to [KVM Device Plugin].(https://github.com/kubevirt/kubernetes-device-plugins/blob/master/docs/README.kvm.md).</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/kubevirt/kubernetes-device-plugins/master/manifests/kvm-ds.yml\n</code></pre> <p>Create NetworkAttachmentDefinition:</p> <pre><code>apiVersion: k8s.cni.cncf.io/v1\nkind: NetworkAttachmentDefinition\nmetadata:\nname: ovn-dpdk\nnamespace: default\nspec:\nconfig: &gt;-\n{\n\"cniVersion\": \"0.3.0\", \n\"type\": \"kube-ovn\", \n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\", \n\"provider\": \"ovn-dpdk.default.ovn\",\n\"vhost_user_socket_volume_name\": \"vhostuser-sockets\",\n\"vhost_user_socket_name\": \"sock\"\n}\n</code></pre> <p>Create a VM image using the following Dockerfile:</p> <pre><code>FROM quay.io/kubevirt/virt-launcher:v0.46.1\n\n# wget http://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud.qcow2\nCOPY CentOS-7-x86_64-GenericCloud.qcow2 /var/lib/libvirt/images/CentOS-7-x86_64-GenericCloud.qcow2\n</code></pre> <p>Create a virtual machine:</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: vm-config\ndata:\nstart.sh: |\nchmod u+w /etc/libvirt/qemu.conf\necho \"hugetlbfs_mount = \\\"/dev/hugepages\\\"\" &gt;&gt; /etc/libvirt/qemu.conf\nvirtlogd &amp;\nlibvirtd &amp;\n\nmkdir /var/lock\n\nsleep 5\n\nvirsh define /root/vm/vm.xml\nvirsh start vm\n\ntail -f /dev/null\nvm.xml: |\n&lt;domain type='kvm'&gt;\n&lt;name&gt;vm&lt;/name&gt;\n&lt;uuid&gt;4a9b3f53-fa2a-47f3-a757-dd87720d9d1d&lt;/uuid&gt;\n&lt;memory unit='KiB'&gt;2097152&lt;/memory&gt;\n&lt;currentMemory unit='KiB'&gt;2097152&lt;/currentMemory&gt;\n&lt;memoryBacking&gt;\n&lt;hugepages&gt;\n&lt;page size='2' unit='M' nodeset='0'/&gt;\n&lt;/hugepages&gt;\n&lt;/memoryBacking&gt;\n&lt;vcpu placement='static'&gt;2&lt;/vcpu&gt;\n&lt;cputune&gt;\n&lt;shares&gt;4096&lt;/shares&gt;\n&lt;vcpupin vcpu='0' cpuset='4'/&gt;\n&lt;vcpupin vcpu='1' cpuset='5'/&gt;\n&lt;emulatorpin cpuset='1,3'/&gt;\n&lt;/cputune&gt;\n&lt;os&gt;\n&lt;type arch='x86_64' machine='pc'&gt;hvm&lt;/type&gt;\n&lt;boot dev='hd'/&gt;\n&lt;/os&gt;\n&lt;features&gt;\n&lt;acpi/&gt;\n&lt;apic/&gt;\n&lt;/features&gt;\n&lt;cpu mode='host-model'&gt;\n&lt;model fallback='allow'/&gt;\n&lt;topology sockets='1' cores='2' threads='1'/&gt;\n&lt;numa&gt;\n&lt;cell id='0' cpus='0-1' memory='2097152' unit='KiB' memAccess='shared'/&gt;\n&lt;/numa&gt;\n&lt;/cpu&gt;\n&lt;on_reboot&gt;restart&lt;/on_reboot&gt;\n&lt;devices&gt;\n&lt;emulator&gt;/usr/libexec/qemu-kvm&lt;/emulator&gt;\n&lt;disk type='file' device='disk'&gt;\n&lt;driver name='qemu' type='qcow2' cache='none'/&gt;\n&lt;source file='/var/lib/libvirt/images/CentOS-7-x86_64-GenericCloud.qcow2'/&gt;\n&lt;target dev='vda' bus='virtio'/&gt;\n&lt;/disk&gt;\n\n&lt;interface type='vhostuser'&gt;\n&lt;mac address='00:00:00:0A:30:89'/&gt;\n&lt;source type='unix' path='/var/run/vm/sock' mode='server'/&gt;\n&lt;model type='virtio'/&gt;\n&lt;driver queues='2'&gt;\n&lt;host mrg_rxbuf='off'/&gt;\n&lt;/driver&gt;\n&lt;/interface&gt;\n&lt;serial type='pty'&gt;\n&lt;target type='isa-serial' port='0'&gt;\n&lt;model name='isa-serial'/&gt;\n&lt;/target&gt;\n&lt;/serial&gt;\n&lt;console type='pty'&gt;\n&lt;target type='serial' port='0'/&gt;\n&lt;/console&gt;\n&lt;channel type='unix'&gt;\n&lt;source mode='bind' path='/var/lib/libvirt/qemu/channel/target/domain-1-vm/org.qemu.guest_agent.0'/&gt;\n&lt;target type='virtio' name='org.qemu.guest_agent.0' state='connected'/&gt;\n&lt;alias name='channel0'/&gt;\n&lt;address type='virtio-serial' controller='0' bus='0' port='1'/&gt;\n&lt;/channel&gt;\n\n&lt;/devices&gt;\n&lt;/domain&gt;\n---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\nname: vm-deployment\nlabels:\napp: vm\nspec:\nreplicas: 1\nselector:\nmatchLabels:\napp: vm\ntemplate:\nmetadata:\nlabels:\napp: vm\nannotations:\nk8s.v1.cni.cncf.io/networks: default/ovn-dpdk\novn-dpdk.default.ovn.kubernetes.io/ip_address: 10.16.0.96\novn-dpdk.default.ovn.kubernetes.io/mac_address: 00:00:00:0A:30:89\nspec:\nnodeSelector:\novn.kubernetes.io/ovs_dp_type: userspace\nsecurityContext:\nrunAsUser: 0\nvolumes:\n- name: vhostuser-sockets\nemptyDir: {}\n- name: xml\nconfigMap:\nname: vm-config\n- name: hugepage\nemptyDir:\nmedium: HugePages-2Mi\n- name: libvirt-runtime\nemptyDir: {}\ncontainers:\n- name: vm\nimage: vm-vhostuser:latest\ncommand: [\"bash\", \"/root/vm/start.sh\"]\nsecurityContext:\ncapabilities:\nadd:\n- NET_BIND_SERVICE\n- SYS_NICE\n- NET_RAW\n- NET_ADMIN\nprivileged: false\nrunAsUser: 0\nresources:\nlimits:\ncpu: '2'\ndevices.kubevirt.io/kvm: '1'\nmemory: '8784969729'\nhugepages-2Mi: 2Gi\nrequests:\ncpu: 666m\ndevices.kubevirt.io/kvm: '1'\nephemeral-storage: 50M\nmemory: '4490002433'\nvolumeMounts:\n- name: vhostuser-sockets\nmountPath: /var/run/vm\n- name: xml\nmountPath: /root/vm/\n- mountPath: /dev/hugepages\nname: hugepage\n- name: libvirt-runtime\nmountPath: /var/run/libvirt\n</code></pre> <p>Wait for the virtual machine to be created successfully and then go to the Pod to configure the virtual machine:</p> <pre><code># virsh set-user-password vm root 12345\nPassword set successfully for root in vm\n\n# virsh console vm\nConnected to domain 'vm'\nEscape character is ^] (Ctrl + ])\n\nCentOS Linux 7 (Core)\nKernel 3.10.0-1127.el7.x86_64 on an x86_64\n\nlocalhost login: root\nPassword:\nLast login: Fri Feb 25 09:52:54 on ttyS0\n</code></pre> <p>Next, you can log into the virtual machine for network configuration and test:</p> <pre><code>ip link set eth0 mtu 1400\nip addr add 10.16.0.96/16 dev eth0\nip ro add default via 10.16.0.1\nping 114.114.114.114\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/external-gateway/","title":"External Gateway","text":"<p>In some scenarios, all container traffic access to the outside needs to be managed and audited through an external gateway. Kube-OVN can forward outbound traffic to the corresponding external gateway by configuring the appropriate routes in the subnet.</p>"},{"location":"en/advance/external-gateway/#usage","title":"Usage","text":"<pre><code>kind: Subnet\napiVersion: kubeovn.io/v1\nmetadata:\nname: external\nspec:\ncidrBlock: 172.31.0.0/16\ngatewayType: centralized\nnatOutgoing: false\nexternalEgressGateway: 192.168.0.1\npolicyRoutingTableID: 1000\npolicyRoutingPriority: 1500\n</code></pre> <ul> <li><code>natOutgoing</code>: needs to be set to <code>false</code>.</li> <li><code>externalEgressGateway</code>: Set to the address of the external gateway, which needs to be in the same Layer 2 reachable domain as the gateway node.</li> <li><code>policyRoutingTableID</code>: The TableID of the local policy routing table used needs to be different for each subnet to avoid conflicts.</li> <li><code>policyRoutingPriority</code>: Route priority, in order to avoid subsequent user customization of other routing operations conflict, here you can specify the route priority. If no special needs, you can fill in any value.</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/fastpath/","title":"Compile FastPath Module","text":"<p>After a data plane performance profile, <code>netfilter</code> consumes about 20% of CPU resources for related processing within the container and on the host. The FastPath module can bypass <code>netfilter</code> to reduce CPU consumption and latency, and increase throughput. This document will describe how to compile the FastPath module manually.</p>"},{"location":"en/advance/fastpath/#download-related-code","title":"Download Related Code","text":"<pre><code>git clone --depth=1 https://github.com/kubeovn/kube-ovn.git\n</code></pre>"},{"location":"en/advance/fastpath/#install-dependencies","title":"Install Dependencies","text":"<p>Here is an example of CentOS dependencies to download:</p> <pre><code>yum install -y kernel-devel-$(uname -r) gcc elfutils-libelf-devel\n</code></pre>"},{"location":"en/advance/fastpath/#compile-the-module","title":"Compile the Module","text":"<p>For the 3.x kernel:</p> <pre><code>cd kube-ovn/fastpath\nmake all\n</code></pre> <p>For the 4.x kernel:</p> <pre><code>cd kube-ovn/fastpath/4.18\ncp ../Makefile .\nmake all\n</code></pre>"},{"location":"en/advance/fastpath/#instal-the-kernel-module","title":"Instal the Kernel Module","text":"<p>Copy <code>kube_ovn_fastpath.ko</code> to each node that needs performance optimization, and run the following command:</p> <pre><code>insmod kube_ovn_fastpath.ko\n</code></pre> <p>Use <code>dmesg</code> to confirm successful installation:</p> <pre><code># dmesg\n[619631.323788] init_module,kube_ovn_fastpath_local_out\n[619631.323798] init_module,kube_ovn_fastpath_post_routing\n[619631.323800] init_module,kube_ovn_fastpath_pre_routing\n[619631.323801] init_module,kube_ovn_fastpath_local_in\n</code></pre> <p>To uninstall a module, use the following command.</p> <pre><code>rmmod kube_ovn_fastpath.ko\n</code></pre> <p>This module will not be loaded automatically after machine reboot. If you want to load it automatically, please write the corresponding autostart script according to the system configuration.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/multi-nic/","title":"Manage Multiple Interface","text":"<p>Kube-OVN can provide cluster-level IPAM capabilities for other CNI network plugins such as macvlan, vlan, host-device, etc. Other network plugins can then use the subnet and fixed IP capabilities in Kube-OVN.</p> <p>Kube-OVN also supports address management when multiple NICs are all of Kube-OVN type.</p>"},{"location":"en/advance/multi-nic/#working-principle","title":"Working Principle","text":"<p>By using Multus CNI, we can add multiple NICs of different networks to a Pod. However, we still lack the ability to manage the IP addresses of different networks within a cluster. In Kube-OVN, we have been able to perform advanced IP management such as subnet management, IP reservation, random assignment, fixed assignment, etc. through CRD of Subnet and IP. Now Kube-OVN extend the subnet to integrate with other different network plugins, so that other network plugins can also use the IPAM functionality of Kube-OVN.</p>"},{"location":"en/advance/multi-nic/#workflow","title":"Workflow","text":"<p>The above diagram shows how to manage the IP addresses of other network plugins via Kube-OVN. The eth0 NIC of the container is connected to the OVN network and the net1 NIC is connected to other CNI networks. The network definition for the net1 network is taken from the NetworkAttachmentDefinition resource definition in multus-cni.</p> <p>When a Pod is created, <code>kube-ovn-controller</code> will get the Pod add event, find the corresponding Subnet according to the annotation in the Pod, then manage the address from it, and write the address information assigned to the Pod back to the Pod annotation.</p> <p>The CNI on the container machine can configure <code>kube-ovn-cni</code> as the ipam plugin. <code>kube-ovn-cni</code> will read the Pod annotation and return the address information to the corresponding CNI plugin using the standard format of the CNI protocol.</p>"},{"location":"en/advance/multi-nic/#usage","title":"Usage","text":""},{"location":"en/advance/multi-nic/#install-kube-ovn-and-multus","title":"Install Kube-OVN and Multus","text":"<p>Please refer One-Click Installation and Multus how to use to install Kube-OVN and Multus-CNI.</p>"},{"location":"en/advance/multi-nic/#provide-ipam-for-other-types-of-cni","title":"Provide IPAM for other types of CNI","text":""},{"location":"en/advance/multi-nic/#create-networkattachmentdefinition","title":"Create NetworkAttachmentDefinition","text":"<p>Here we use macvlan as the second network of the container network and set its ipam to <code>kube-ovn</code>:</p> <pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: macvlan\nnamespace: default\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.0\",\n\"type\": \"macvlan\",\n\"master\": \"eth0\",\n\"mode\": \"bridge\",\n\"ipam\": {\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"macvlan.default\"\n}\n}'\n</code></pre> <ul> <li><code>spec.config.ipam.type</code>: Need to be set to <code>kube-ovn</code> to call the kube-ovn plugin to get the address information.</li> <li><code>server_socket</code>: The socket file used for communication to Kube-OVN. The default location is <code>/run/openvswitch/kube-ovn-daemon.sock</code>.</li> <li><code>provider</code>: The current NetworkAttachmentDefinition's <code>&lt;name&gt;. &lt;namespace&gt;</code> , Kube-OVN will use this information to find the corresponding Subnet resource.</li> </ul>"},{"location":"en/advance/multi-nic/#the-attached-nic-is-a-kube-ovn-type-nic","title":"The attached NIC is a Kube-OVN type NIC","text":"<p>At this point, the multiple NICs are all Kube-OVN type NICs.</p>"},{"location":"en/advance/multi-nic/#create-networkattachmentdefinition_1","title":"Create NetworkAttachmentDefinition","text":"<p>Set the <code>provider</code> suffix to <code>ovn</code>:</p> <pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: attachnet\nnamespace: default\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.0\",\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"attachnet.default.ovn\"\n}'\n</code></pre> <ul> <li><code>spec.config.ipam.type</code>: Need to be set to <code>kube-ovn</code> to call the kube-ovn plugin to get the address information.</li> <li><code>server_socket</code>: The socket file used for communication to Kube-OVN. The default location is <code>/run/openvswitch/kube-ovn-daemon.sock</code>.</li> <li><code>provider</code>: The current NetworkAttachmentDefinition's <code>&lt;name&gt;. &lt;namespace&gt;</code> , Kube-OVN will use this information to find the corresponding Subnet resource. It should have the suffix <code>ovn</code> here.</li> </ul>"},{"location":"en/advance/multi-nic/#create-a-kube-ovn-subnet","title":"Create a Kube-OVN Subnet","text":"<p>Create a Kube-OVN Subnet, set the corresponding <code>cidrBlock</code> and <code>exclude_ips</code>, the <code>provider</code> should be set to the <code>&lt;name&gt;. &lt;namespace&gt;</code> of corresponding NetworkAttachmentDefinition. For example, to provide additional NICs with macvlan, create a Subnet as follows:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: macvlan\nspec:\nprotocol: IPv4\nprovider: macvlan.default\ncidrBlock: 172.17.0.0/16\ngateway: 172.17.0.1\nexcludeIps:\n- 172.17.0.0..172.17.0.10\n</code></pre> <p><code>gateway</code>, <code>private</code>, <code>nat</code> are only valid for networks with <code>provider</code> type ovn, not for attachment networks.</p> <p>If you are using Kube-OVN as an attached NIC, <code>provider</code> should be set to the <code>&lt;name&gt;. &lt;namespace&gt;.ovn</code> of the corresponding NetworkAttachmentDefinition, and should end with <code>ovn</code> as a suffix.</p> <p>An example of creating a Subnet with an additional NIC provided by Kube-OVN is as follows:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: attachnet\nspec:\nprotocol: IPv4\nprovider: attachnet.default.ovn\ncidrBlock: 172.17.0.0/16\ngateway: 172.17.0.1\nexcludeIps:\n- 172.17.0.0..172.17.0.10\n</code></pre>"},{"location":"en/advance/multi-nic/#create-a-pod-with-multiple-nic","title":"Create a Pod with Multiple NIC","text":"<p>For Pods with randomly assigned addresses, simply add the following annotation <code>k8s.v1.cni.cncf.io/networks</code>, taking the value <code>&lt;namespace&gt;/&lt;name&gt;</code> of the corresponding NetworkAttachmentDefinition.\uff1a</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: samplepod\nnamespace: default\nannotations:\nk8s.v1.cni.cncf.io/networks: default/macvlan\nspec:\ncontainers:\n- name: samplepod\ncommand: [\"/bin/ash\", \"-c\", \"trap : TERM INT; sleep infinity &amp; wait\"]\nimage: docker.io/library/alpine:edge\n</code></pre>"},{"location":"en/advance/multi-nic/#create-pod-with-a-fixed-ip","title":"Create Pod with a Fixed IP","text":"<p>For Pods with fixed IPs, add <code>&lt;networkAttachmentName&gt;.&lt;networkAttachmentNamespace&gt;.kubernetes.io/ip_address</code> annotation\uff1a</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: static-ip\nnamespace: default\nannotations:\nk8s.v1.cni.cncf.io/networks: default/macvlan\novn.kubernetes.io/ip_address: 10.16.0.15\novn.kubernetes.io/mac_address: 00:00:00:53:6B:B6\nmacvlan.default.kubernetes.io/ip_address: 172.17.0.100\nmacvlan.default.kubernetes.io/mac_address: 00:00:00:53:6B:BB\nspec:\ncontainers:\n- name: static-ip\nimage: docker.io/library/nginx:alpine\n</code></pre>"},{"location":"en/advance/multi-nic/#create-workloads-with-fixed-ips","title":"Create Workloads with Fixed IPs","text":"<p>For workloads that use ippool, add <code>&lt;networkAttachmentName&gt;.&lt;networkAttachmentNamespace&gt;.kubernetes.io/ip_pool</code> annotations:</p> <pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nnamespace: default\nname: static-workload\nlabels:\napp: static-workload\nspec:\nreplicas: 2\nselector:\nmatchLabels:\napp: static-workload\ntemplate:\nmetadata:\nlabels:\napp: static-workload\nannotations:\nk8s.v1.cni.cncf.io/networks: default/macvlan\novn.kubernetes.io/ip_pool: 10.16.0.15,10.16.0.16,10.16.0.17\nmacvlan.default.kubernetes.io/ip_pool: 172.17.0.200,172.17.0.201,172.17.0.202\nspec:\ncontainers:\n- name: static-workload\nimage: docker.io/library/nginx:alpine\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/nat-policy-rule/","title":"Default VPC NAT Policy Rule","text":""},{"location":"en/advance/nat-policy-rule/#purpose","title":"Purpose","text":"<p>In the Overlay Subnet under the default VPC, when the <code>natOutgoing</code> switch is turned on, all Pods in the subnet need to do SNAT to access the external network, but in some scenarios we do not want all Pods in the subnet to access the external network by SNAT.</p> <p>So the NAT Policy Rule is to provide a way for users to decide which CIDRs or IPs in the subnet to access the external network need SNAT.</p>"},{"location":"en/advance/nat-policy-rule/#how-to-use-nat-policy-rules","title":"How to use NAT Policy Rules","text":"<p>Enable the <code>natOutgoing</code> switch in <code>subnet.Spec</code>, and add the field <code>natOutgoingPolicyRules</code> as follows:</p> <pre><code>spec:\nnatOutgoing: true\nnatOutgoingPolicyRules:\n- action: forward\nmatch:\nsrcIPs: 10.0.11.0/30,10.0.11.254\n- action: nat\nmatch:\nsrcIPs: 10.0.11.128/26\ndstIPs: 114.114.114.114,8.8.8.8\n</code></pre> <p>The above case shows that there are two NAT policy rules:</p> <ol> <li>Packets with source IP 10.0.11.0/30 or 10.0.11.254 will not perform SNAT when accessing the external network.</li> <li>When a packet with source IP 10.0.11.128/26 and destination IP 114.114.114.114 or 8.8.8.8 accesses the external network, SNAT will be performed.</li> </ol> <p>Field description:</p> <p><code>action</code>: The action that will be executed for packets that meets the corresponding conditions of the <code>match</code>. The action is divided into two types: <code>forward</code> and <code>nat</code>. When natOutgoingPolicyRules is not configured, packets are still SNAT by default.</p> <p><code>match</code>: Indicates the matching segment of the message, the matching segment includes <code>srcIPs</code> and <code>dstIPs</code>, here indicates the source IP and destination IP of the message from the subnet to the external network. <code>match.srcIPs</code> and <code>match.dstIPs</code> support multiple cidr and ip, separated by commas. If multiple match rules overlap, the action that is matched first will be executed according to the order of the <code>natOutgoingPolicyRules</code> array.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/node-local-dns/","title":"NodeLocal DNSCache and Kube-OVN adaptation","text":"<p>NodeLocal DNSCache improves cluster DNS performance by running DNS cache as a DaemonSet on cluster nodes. This function can also be adapted to Kube-OVN.</p>"},{"location":"en/advance/node-local-dns/#nodelocal-dnscache-deployment","title":"Nodelocal DNSCache deployment","text":""},{"location":"en/advance/node-local-dns/#deploy-kubernetes-nodelocal-dnscache","title":"Deploy Kubernetes NodeLocal DNScache","text":"<p>This step refers to Kubernetes official website configuration nodelocaldnscache.</p> <p>Deploy with the following script:</p> <pre><code>#!bin/bash\n\nlocaldns=169.254.20.10\ndomain=cluster.local\nkubedns=10.96.0.10\n\nwget https://raw.githubusercontent.com/kubernetes/kubernetes/master/cluster/addons/dns/nodelocaldns/nodelocaldns.yaml\nsed -i \"s/__PILLAR__LOCAL__DNS__/$localdns/g; s/__PILLAR__DNS__DOMAIN__/$domain/g; s/,__PILLAR__DNS__SERVER__//g; s/__PILLAR__CLUSTER__DNS__/$kubedns/g\" nodelocaldns.yaml\n\nkubectl apply -f nodelocaldns.yaml\n</code></pre> <p>Modify the kubelet configuration file on each node, modify the clusterDNS field in <code>/var/lib/kubelet/config.yaml</code> to the local DNS IP 169.254.20.10, and then restart the kubelet service.</p>"},{"location":"en/advance/node-local-dns/#kube-ovn-corresponding-dns-configuration","title":"Kube-OVN corresponding DNS configuration","text":"<p>After deploying the Nodelocal DNScache component of Kubernetes, Kube-OVN needs to make the following modifications:</p>"},{"location":"en/advance/node-local-dns/#underlay-subnet-enable-u2o-switch","title":"Underlay subnet enable U2O switch","text":"<p>If the underlay subnet needs to use the local DNS function, you need to enable the U2O function, that is, configure <code>spec.u2oInterconnection = true</code> in <code>kubectl edit subnet {your subnet}</code>. If it is an overlay subnet, this step is not required.</p>"},{"location":"en/advance/node-local-dns/#specify-the-corresponding-local-dns-ip-for-kube-ovn-controller","title":"Specify the corresponding local DNS IP for kube-ovn-controller","text":"<pre><code>kubectl edit deployment kube-ovn-controller -n kube-system\n</code></pre> <p>Add field to spec.template.spec.containers.args <code>--node-local-dns-ip=169.254.20.10</code></p>"},{"location":"en/advance/node-local-dns/#rebuild-the-created-pods","title":"Rebuild the created Pods","text":"<p>The reason for this step is to let the Pod regenerate <code>/etc/resolv.conf</code> so that the nameserver points to the local DNS IP. If the nameserver of the Pod is not rebuilt, it will still use the DNS ClusterIP of the cluster. At the same time, if the u2o switch is turned on, the Pod needs to be rebuilt to regenerate the Pod gateway.</p>"},{"location":"en/advance/node-local-dns/#validator-local-dns-cache-function","title":"Validator local DNS cache function","text":"<p>After the above configuration is completed, you can find the Pod verification as follows. You can see that the Pod's DNS server points to the local 169.254.20.10 and successfully resolves the domain name:</p> <pre><code># kubectl exec -it pod1 -- nslookup github.com\nServer:         169.254.20.10\nAddress:        169.254.20.10:53\n\n\nName:   github.com\nAddress: 20.205.243.166\n</code></pre> <p>You can also capture packets at the node and verify as follows. You can see that the DNS query message reaches the local DNS service through the ovn0 network card, and the DNS response message returns in the same way:</p> <pre><code># tcpdump -i any port 53\n\n06:20:00.441889 659246098c56_h P   ifindex 17 00:00:00:73:f1:06 ethertype IPv4 (0x0800), length 75: 10.16.0.2.40230 &gt; 169.254.20.10.53: 1291+ A? baidu.com. (27)\n06:20:00.441889 ovn0  In  ifindex 7 00:00:00:50:32:cd ethertype IPv4 (0x0800), length 75: 10.16.0.2.40230 &gt; 169.254.20.10.53: 1291+ A? baidu.com. (27)\n06:20:00.441950 659246098c56_h P   ifindex 17 00:00:00:73:f1:06 ethertype IPv4 (0x0800), length 75: 10.16.0.2.40230 &gt; 169.254.20.10.53: 1611+ AAAA? baidu.com. (27)\n06:20:00.441950 ovn0  In  ifindex 7 00:00:00:50:32:cd ethertype IPv4 (0x0800), length 75: 10.16.0.2.40230 &gt; 169.254.20.10.53: 1611+ AAAA? baidu.com. (27)\n06:20:00.442203 ovn0  Out ifindex 7 00:00:00:52:99:d8 ethertype IPv4 (0x0800), length 145: 169.254.20.10.53 &gt; 10.16.0.2.40230: 1611* 0/1/0 (97)\n06:20:00.442219 659246098c56_h Out ifindex 17 00:00:00:ea:b3:5e ethertype IPv4 (0x0800), length 145: 169.254.20.10.53 &gt; 10.16.0.2.40230: 1611* 0/1/0 (97)\n06:20:00.442273 ovn0  Out ifindex 7 00:00:00:52:99:d8 ethertype IPv4 (0x0800), length 125: 169.254.20.10.53 &gt; 10.16.0.2.40230: 1291* 2/0/0 A 39.156.66.10, A 110.242.68.66 (77)\n06:20:00.442278 659246098c56_h Out ifindex 17 00:00:00:ea:b3:5e ethertype IPv4 (0x0800), length 125: 169.254.20.10.53 &gt; 10.16.0.2.40230: 1291* 2/0/0 A 39.156.66.10, A 110.242.68.66 (77)\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/offload-corigine/","title":"Offload with Corigine","text":"<p>Kube-OVN uses OVS for traffic forwarding in the final data plane, and the associated flow table matching, tunnel encapsulation and other functions are CPU-intensive, which consumes a lot of CPU resources and leads to higher latency and lower throughput under heavy traffic. Corigine Agilio CX series SmartNIC can offload OVS-related operations to the hardware. This technology can shorten the data path without modifying the OVS control plane, avoiding the use of host CPU resources, which dramatically reduce latency and significantly increase the throughput.</p> <p></p>"},{"location":"en/advance/offload-corigine/#prerequisites","title":"Prerequisites","text":"<ul> <li>Corigine Agilio CX series SmartNIC.</li> <li>CentOS 8 Stream or Linux 5.7 above.</li> <li>Since the current NIC does not support <code>dp_hash</code> and <code>hash</code> operation offload, OVN LB function should be disabled.</li> </ul>"},{"location":"en/advance/offload-corigine/#setup-sr-iov","title":"Setup SR-IOV","text":"<p>Please read Agilio Open vSwitch TC User Guide for the detail usage of this SmartNIC.</p> <p>The following scripts are saved for subsequent execution of firmware-related operations:</p> <pre><code>#!/bin/bash\nDEVICE=${1}\nDEFAULT_ASSY=scan\nASSY=${2:-${DEFAULT_ASSY}}\nAPP=${3:-flower}\n\nif [ \"x${DEVICE}\" = \"x\" -o ! -e /sys/class/net/${DEVICE} ]; then\necho Syntax: ${0} device [ASSY] [APP]\necho\necho This script associates the TC Offload firmware\n    echo with a Netronome SmartNIC.\n    echo\necho device: is the network device associated with the SmartNIC\n    echo ASSY: defaults to ${DEFAULT_ASSY}\necho APP: defaults to flower. flower-next is supported if updated\n    echo      firmware has been installed.\n    exit 1\nfi\n\n# It is recommended that the assembly be determined by inspection\n# The following code determines the value via the debug interface\nif [ \"${ASSY}x\" = \"scanx\" ]; then\nethtool -W ${DEVICE} 0\nDEBUG=$(ethtool -w ${DEVICE} data /dev/stdout | strings)\nSERIAL=$(echo \"${DEBUG}\" | grep \"^SN:\")\nASSY=$(echo ${SERIAL} | grep -oE AMDA[0-9]{4})\nfi\n\nPCIADDR=$(basename $(readlink -e /sys/class/net/${DEVICE}/device))\nFWDIR=\"/lib/firmware/netronome\"\n\n# AMDA0081 and AMDA0097 uses the same firmware\nif [ \"${ASSY}\" = \"AMDA0081\" ]; then\nif [ ! -e ${FWDIR}/${APP}/nic_AMDA0081.nffw ]; then\nln -sf nic_AMDA0097.nffw ${FWDIR}/${APP}/nic_AMDA0081.nffw\n   fi\nfi\n\nFW=\"${FWDIR}/pci-${PCIADDR}.nffw\"\nln -sf \"${APP}/nic_${ASSY}.nffw\" \"${FW}\"\n\n# insert distro-specific initramfs section here...\n</code></pre> <p>Switching firmware options and reloading the driver:</p> <pre><code>./agilio-tc-fw-select.sh ens47np0 scan\nrmmod nfp\nmodprobe nfp\n</code></pre> <p>Check the number of available VFs and create VFs.</p> <pre><code># cat /sys/class/net/ens3/device/sriov_totalvfs\n65\n\n# echo 4 &gt; /sys/class/net/ens47/device/sriov_numvfs\n</code></pre>"},{"location":"en/advance/offload-corigine/#install-sr-iov-device-plugin","title":"Install SR-IOV Device Plugin","text":"<p>Since each machine has a limited number of VFs and each Pod that uses acceleration will take up VF resources, we need to use the SR-IOV Device Plugin to manage the corresponding resources so that the scheduler knows how to schedule.</p> <p>Create SR-IOV Configmap:</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: sriovdp-config\nnamespace: kube-system\ndata:\nconfig.json: |\n{\n\"resourceList\": [{\n\"resourcePrefix\": \"corigine.com\",\n\"resourceName\": \"agilio_sriov\",\n\"selectors\": {\n\"vendors\": [\"19ee\"],\n\"devices\": [\"6003\"],\n\"drivers\": [\"nfp_netvf\"]\n}\n}\n]\n}\n</code></pre> <p>Please read the SR-IOV device plugin to deploy:</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/intel/sriov-network-device-plugin/master/deployments/k8s-v1.16/sriovdp-daemonset.yaml\n</code></pre> <p>Check if SR-IOV resources have been registered to Kubernetes Node:</p> <pre><code>kubectl describe no containerserver  | grep corigine\n\ncorigine.com/agilio_sriov:  4\ncorigine.com/agilio_sriov:  4\ncorigine.com/agilio_sriov  0           0\n</code></pre>"},{"location":"en/advance/offload-corigine/#install-multus-cni","title":"Install Multus-CNI","text":"<p>The device IDs obtained during SR-IOV Device Plugin scheduling need to be passed to Kube-OVN via Multus-CNI, so Multus-CNI needs to be configured to perform the related tasks.</p> <p>Please read Multus-CNI Document to deploy\uff1a</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml\n</code></pre> <p>Create <code>NetworkAttachmentDefinition</code>\uff1a</p> <pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: default\nnamespace: default\nannotations:\nk8s.v1.cni.cncf.io/resourceName: corigine.com/agilio_sriov\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.1\",\n\"name\": \"kube-ovn\",\n\"plugins\":[\n{\n\"type\":\"kube-ovn\",\n\"server_socket\":\"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"default.default.ovn\"\n},\n{\n\"type\":\"portmap\",\n\"capabilities\":{\n\"portMappings\":true\n}\n}\n]\n}'\n</code></pre> <ul> <li><code>provider</code>: the format should be {name}.{namespace}.ovn of related <code>NetworkAttachmentDefinition</code>.</li> </ul>"},{"location":"en/advance/offload-corigine/#enable-offload-in-kube-ovn","title":"Enable Offload in Kube-OVN","text":"<p>Download the scripts:</p> <pre><code>wget https://raw.githubusercontent.com/alauda/kube-ovn/master/dist/images/install.sh\n</code></pre> <p>Change the related options\uff0c<code>IFACE</code> should be the physic NIC and has an IP:</p> <pre><code>ENABLE_MIRROR=${ENABLE_MIRROR:-false}\nHW_OFFLOAD=${HW_OFFLOAD:-true}\nENABLE_LB=${ENABLE_LB:-false}\nIFACE=\"ensp01\"\n</code></pre> <p>Install Kube-OVN\uff1a</p> <pre><code>bash install.sh\n</code></pre>"},{"location":"en/advance/offload-corigine/#create-pods-with-vf-nics","title":"Create Pods with VF NICs","text":"<p>Pods that use VF for network offload acceleration can be created using the following yaml:</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: nginx\nnamespace: default\nannotations:\nv1.multus-cni.io/default-network: default/default\nspec:\ncontainers:\n- name: nginx\nimage: docker.io/library/nginx:alpine\nresources:\nrequests:\ncorigine.com/agilio_sriov: '1'\nlimits:\ncorigine.com/agilio_sriov: '1'\n</code></pre> <ul> <li><code>v1.multus-cni.io/default-network</code>: should be the {namespace}/{name} of related <code>NetworkAttachmentDefinition</code>.</li> </ul> <p>Running the following command in the <code>ovs-ovn</code> container of the Pod run node to observe if offload success.</p> <pre><code># ovs-appctl dpctl/dump-flows -m type=offloaded\nufid:91cc45de-e7e9-4935-8f82-1890430b0f66, skb_priority(0/0),skb_mark(0/0),ct_state(0/0x23),ct_zone(0/0),ct_mark(0/0),ct_label(0/0x1),recirc_id(0),dp_hash(0/0),in_port(5b45c61b307e_h),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:c5:6d:4e,dst=00:00:00:e7:16:ce),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:941539, bytes:62142230, used:0.260s, offloaded:yes, dp:tc, actions:54235e5753b8_h\nufid:e00768d7-e652-4d79-8182-3291d852b791, skb_priority(0/0),skb_mark(0/0),ct_state(0/0x23),ct_zone(0/0),ct_mark(0/0),ct_label(0/0x1),recirc_id(0),dp_hash(0/0),in_port(54235e5753b8_h),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:e7:16:ce,dst=00:00:00:c5:6d:4e),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:82386659, bytes:115944854173, used:0.260s, offloaded:yes, dp:tc, actions:5b45c61b307e_h\n</code></pre> <p>If there is <code>offloaded:yes, dp:tc</code> content, the offloading is successful.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/offload-mellanox/","title":"Offload with Mellanox","text":"<p>Kube-OVN uses OVS for traffic forwarding in the final data plane, and the associated flow table matching, tunnel encapsulation and other functions are CPU-intensive, which consumes a lot of CPU resources and leads to higher latency and lower throughput under heavy traffic. Mellanox Accelerated Switching And Packet Processing (ASAP\u00b2) technology offloads OVS-related operations to an eSwitch within the eSwitch in the hardware. This technology can shorten the data path without modifying the OVS control plane, avoiding the use of host CPU resources, which dramatically reduce latency and significantly increase the throughput.</p> <p></p>"},{"location":"en/advance/offload-mellanox/#prerequisites","title":"Prerequisites","text":"<ul> <li>Mellanox CX5/CX6/BlueField that support ASAP\u00b2.</li> <li>CentOS 8 Stream or Linux 5.7 above.</li> <li>Since the current NIC does not support <code>dp_hash</code> and <code>hash</code> operation offload, OVN LB function should be disabled.</li> <li>In order to support offload mode, the NIC cannot do bond.</li> </ul>"},{"location":"en/advance/offload-mellanox/#setup-sr-iov","title":"Setup SR-IOV","text":"<p>Check the device ID of the NIC, in the following example it is <code>42:00.0</code>:</p> <pre><code># lspci -nn | grep ConnectX-5\n42:00.0 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5] [15b3:1017]\n</code></pre> <p>Find the corresponding NIC by its device ID:</p> <pre><code># ls -l /sys/class/net/ | grep 42:00.0\nlrwxrwxrwx. 1 root root 0 Jul 22 23:16 p4p1 -&gt; ../../devices/pci0000:40/0000:40:02.0/0000:42:00.0/net/p4p1\n</code></pre> <p>Check the number of available VFs:</p> <pre><code># cat /sys/class/net/p4p1/device/sriov_totalvfs\n8\n</code></pre> <p>Create VFs and do not exceeding the number found above:</p> <pre><code># echo '4' &gt; /sys/class/net/p4p1/device/sriov_numvfs\n# ip link show p4p1\n10: p4p1: &lt;NO-CARRIER,BROADCAST,MULTICAST,UP&gt; mtu 1500 qdisc mq state DOWN mode DEFAULT group default qlen 1000\nlink/ether b8:59:9f:c1:ec:12 brd ff:ff:ff:ff:ff:ff\n    vf 0 MAC 00:00:00:00:00:00, spoof checking off, link-state auto, trust off, query_rss off\n    vf 1 MAC 00:00:00:00:00:00, spoof checking off, link-state auto, trust off, query_rss off\n    vf 2 MAC 00:00:00:00:00:00, spoof checking off, link-state auto, trust off, query_rss off\n    vf 3 MAC 00:00:00:00:00:00, spoof checking off, link-state auto, trust off, query_rss off\n# ip link set p4p1 up\n</code></pre> <p>Find the device IDs corresponding to the above VFs:</p> <pre><code># lspci -nn | grep ConnectX-5\n42:00.0 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5] [15b3:1017]\n42:00.1 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5] [15b3:1017]\n42:00.2 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n42:00.3 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n42:00.4 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n42:00.5 Ethernet controller [0200]: Mellanox Technologies MT27800 Family [ConnectX-5 Virtual Function] [15b3:1018]\n</code></pre> <p>Unbound the VFs from the driver:</p> <pre><code>echo 0000:42:00.2 &gt; /sys/bus/pci/drivers/mlx5_core/unbind\necho 0000:42:00.3 &gt; /sys/bus/pci/drivers/mlx5_core/unbind\necho 0000:42:00.4 &gt; /sys/bus/pci/drivers/mlx5_core/unbind\necho 0000:42:00.5 &gt; /sys/bus/pci/drivers/mlx5_core/unbind\n</code></pre> <p>Enable eSwitch mode and set up hardware offload:</p> <pre><code>devlink dev eswitch set pci/0000:42:00.0 mode switchdev\nethtool -K enp66s0f0 hw-tc-offload on\n</code></pre> <p>Rebind the driver and complete the VF setup:</p> <pre><code>echo 0000:42:00.2 &gt; /sys/bus/pci/drivers/mlx5_core/bind\necho 0000:42:00.3 &gt; /sys/bus/pci/drivers/mlx5_core/bind\necho 0000:42:00.4 &gt; /sys/bus/pci/drivers/mlx5_core/bind\necho 0000:42:00.5 &gt; /sys/bus/pci/drivers/mlx5_core/bind\n</code></pre> <p>Some behaviors of <code>NetworkManager</code> may cause driver exceptions, if offloading problems occur we recommended to close <code>NetworkManager</code> and try again.</p> <pre><code>systemctl stop NetworkManager\nsystemctl disable NetworkManager\n</code></pre>"},{"location":"en/advance/offload-mellanox/#install-sr-iov-device-plugin","title":"Install SR-IOV Device Plugin","text":"<p>Since each machine has a limited number of VFs and each Pod that uses acceleration will take up VF resources, we need to use the SR-IOV Device Plugin to manage the corresponding resources so that the scheduler knows how to schedule.</p> <p>Create SR-IOV Configmap:</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: sriovdp-config\nnamespace: kube-system\ndata:\nconfig.json: |\n{\n\"resourceList\": [{\n\"resourcePrefix\": \"mellanox.com\",\n\"resourceName\": \"cx5_sriov_switchdev\",\n\"selectors\": {\n\"vendors\": [\"15b3\"],\n\"devices\": [\"1018\"],\n\"drivers\": [\"mlx5_core\"]\n}\n}\n]\n}\n</code></pre> <p>Please read the SR-IOV device plugin to deploy:</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/intel/sriov-network-device-plugin/master/deployments/k8s-v1.16/sriovdp-daemonset.yaml\n</code></pre> <p>Check if SR-IOV resources have been registered to Kubernetes Node:</p> <pre><code>kubectl describe node kube-ovn-01  | grep mellanox\n\nmellanox.com/cx5_sriov_switchdev:  4\nmellanox.com/cx5_sriov_switchdev:  4\nmellanox.com/cx5_sriov_switchdev  0           0\n</code></pre>"},{"location":"en/advance/offload-mellanox/#install-multus-cni","title":"Install Multus-CNI","text":"<p>The device IDs obtained during SR-IOV Device Plugin scheduling need to be passed to Kube-OVN via Multus-CNI, so Multus-CNI needs to be configured to perform the related tasks.</p> <p>Please read Multus-CNI Document to deploy\uff1a</p> <pre><code>kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml\n</code></pre> <p>Create <code>NetworkAttachmentDefinition</code>\uff1a</p> <pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: default\nnamespace: default\nannotations:\nk8s.v1.cni.cncf.io/resourceName: mellanox.com/cx5_sriov_switchdev\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.1\",\n\"name\": \"kube-ovn\",\n\"plugins\":[\n{\n\"type\":\"kube-ovn\",\n\"server_socket\":\"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"default.default.ovn\"\n},\n{\n\"type\":\"portmap\",\n\"capabilities\":{\n\"portMappings\":true\n}\n}\n]\n}'\n</code></pre> <ul> <li><code>provider</code>: the format should be {name}.{namespace}.ovn of related <code>NetworkAttachmentDefinition</code>.</li> </ul>"},{"location":"en/advance/offload-mellanox/#enable-offload-in-kube-ovn","title":"Enable Offload in Kube-OVN","text":"<p>Download the scripts:</p> <pre><code>wget https://raw.githubusercontent.com/alauda/kube-ovn/master/dist/images/install.sh\n</code></pre> <p>Change the related options\uff0c<code>IFACE</code> should be the physic NIC and has an IP:</p> <pre><code>ENABLE_MIRROR=${ENABLE_MIRROR:-false}\nHW_OFFLOAD=${HW_OFFLOAD:-true}\nENABLE_LB=${ENABLE_LB:-false}\nIFACE=\"ensp01\"\n</code></pre> <p>Install Kube-OVN\uff1a</p> <pre><code>bash install.sh\n</code></pre>"},{"location":"en/advance/offload-mellanox/#create-pods-with-vf-nics","title":"Create Pods with VF NICs","text":"<p>Pods that use VF for network offload acceleration can be created using the following yaml:</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: nginx\nannotations:\nv1.multus-cni.io/default-network: default/default\nspec:\ncontainers:\n- name: nginx\nimage: docker.io/library/nginx:alpine\nresources:\nrequests:\nmellanox.com/cx5_sriov_switchdev: '1'\nlimits:\nmellanox.com/cx5_sriov_switchdev: '1'\n</code></pre> <ul> <li><code>v1.multus-cni.io/default-network</code>: should be the {namespace}/{name} of related <code>NetworkAttachmentDefinition</code>.</li> </ul> <p>Running the following command in the <code>ovs-ovn</code> container of the Pod run node to observe if offload success.</p> <pre><code># ovs-appctl dpctl/dump-flows -m type=offloaded\nufid:91cc45de-e7e9-4935-8f82-1890430b0f66, skb_priority(0/0),skb_mark(0/0),ct_state(0/0x23),ct_zone(0/0),ct_mark(0/0),ct_label(0/0x1),recirc_id(0),dp_hash(0/0),in_port(5b45c61b307e_h),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:c5:6d:4e,dst=00:00:00:e7:16:ce),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:941539, bytes:62142230, used:0.260s, offloaded:yes, dp:tc, actions:54235e5753b8_h\nufid:e00768d7-e652-4d79-8182-3291d852b791, skb_priority(0/0),skb_mark(0/0),ct_state(0/0x23),ct_zone(0/0),ct_mark(0/0),ct_label(0/0x1),recirc_id(0),dp_hash(0/0),in_port(54235e5753b8_h),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:e7:16:ce,dst=00:00:00:c5:6d:4e),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:82386659, bytes:115944854173, used:0.260s, offloaded:yes, dp:tc, actions:5b45c61b307e_h\n</code></pre> <p>If there is <code>offloaded:yes, dp:tc</code> content, the offloading is successful.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/overlay-with-route/","title":"Interconnection with Routes in Overlay Mode","text":"<p>In some scenarios, the network environment does not support Underlay mode, but still need Pods and external devices directly access through IP, then you can use the routing method to connect the container network and the external.</p> <p>Only Overlay Subnets in default VPC support this method. In this case, the Pod IP goes directly to the underlying network, which needs to disable IP checks for source and destination addresses.</p>"},{"location":"en/advance/overlay-with-route/#prerequisites","title":"Prerequisites","text":"<ul> <li>In this mode, the host needs to open the <code>ip_forward</code>.</li> <li>Check if there is a <code>Drop</code> rule in the forward chain in the host iptables that should be modified for container-related traffic.</li> <li>Due to the possibility of asymmetric routing, the host needs to allow packets with a ct status of <code>INVALID</code>.</li> </ul>"},{"location":"en/advance/overlay-with-route/#steps","title":"Steps","text":"<p>For subnets that require direct external routing, you need to set <code>natOutgoing</code> of the subnet to <code>false</code> to turn off nat mapping and make the Pod IP directly accessible to the external network.</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: routed\nspec:\nprotocol: IPv4\ncidrBlock: 10.166.0.0/16\ndefault: false\nexcludeIps:\n- 10.166.0.1\ngateway: 10.166.0.1\ngatewayType: distributed\nnatOutgoing: false\n</code></pre> <p>At this point, the Pod's packets can reach the peer node via the host route, but the peer node does not yet know where the return packets should be sent to and needs to add a return route.</p> <p>If the peer host and the container host are on the same Layer 2 network, we can add a static route directly to the peer host to point the next hop of the container network to any machine in the Kubernetes cluster.</p> <pre><code>ip route add 10.166.0.0/16 via 192.168.2.10 dev eth0\n</code></pre> <p><code>10.166.0.0/16</code> is the container subnet CIDR, and <code>192.168.2.10</code> is one node in the Kubernetes cluster.</p> <p>If the peer host and the container host are not in the same layer 2 network, you need to configure the corresponding rules on the router.</p> <p>Note: Specifying an IP for a single node may lead to single point of failure. To achieve fast failover, Keepalived can be used to set up a VIP for multiple nodes, and the next hop of the route can be directed to the VIP.</p> <p>In some virtualized environments, the virtual network identifies asymmetric traffic as illegal traffic and drops it. In this case, you need to adjust the <code>gatewayType</code> of the Subnet to <code>centralized</code> and set the next hop to the IP of the <code>gatewayNode</code> node during route setup.</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: routed\nspec:\nprotocol: IPv4\ncidrBlock: 10.166.0.0/16\ndefault: false\nexcludeIps:\n- 10.166.0.1\ngateway: 10.166.0.1\ngatewayType: centralized\ngatewayNode: \"node1\"\nnatOutgoing: false\n</code></pre> <p>If you still want to perform NAT processing for some traffic, such as traffic accessing the Internet, please refer to the Default VPC NAT Policy Rule.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/ovn-eip-fip-snat/","title":"Support OVN EIP,FIP and SNAT","text":"<pre><code>graph LR\n\n\npod--&gt;vpc1-subnet--&gt;vpc1--&gt;snat--&gt;lrp--&gt;external-subnet--&gt;gw-node-external-nic</code></pre> <p>The pod access the public network based on the snat</p> <pre><code>graph LR\n\n\npod--&gt;vpc1-subnet--&gt;vpc1--&gt;fip--&gt;lrp--&gt;external-subnet--&gt;local-node-external-nic</code></pre> <p>The pod access the public network based on the fip</p> <p>The CRD supported by this function is basically the same as the iptable nat gw public network solution.</p> <ul> <li>ovn eip: occupies a public ip address and is allocated from the underlay provider network vlan subnet</li> <li>ovn fip: one-to-one dnat snat, which provides direct public network access for ip addresses and vip in a vpc</li> <li>ovn snat: a subnet cidr or a single vpc ip or vip can access public networks based on snat</li> <li>ovn dnat: based router lb, which enables direct access to a group of endpoints in a vpc based on a public endpoint</li> </ul>"},{"location":"en/advance/ovn-eip-fip-snat/#1-deployment","title":"1. Deployment","text":"<p>Currently allows all vpcs to share the same provider vlan subnet resources, similar to neutron ovn mode. Compatible with previous scenarios default VPC EIP/SNAT.</p> <p>During the deployment phase, you may need to specify a default public network logical switch based on actual conditions. If no vlan is in use (vlan 0 is used), the following startup parameters do not need to be configured.</p> <pre><code># When deploying you need to refer to the above scenario and specify the following parameters as needed according to the actual situation\n# 1. kube-ovn-controller Startup parameters to be configured\uff1a\n- --external-gateway-vlanid=204\n- --external-gateway-switch=external204\n\n# 2. kube-ovn-cni Startup parameters to be configured:\n- --external-gateway-switch=external204 # The above configuration is consistent with the following public network configuration vlan id and resource name, \n# currently only support to specify one underlay public network as the default external public network.\n</code></pre> <p>The design and use of this configuration item takes into account the following factors\uff1a</p> <ul> <li>Based on this configuration item can be docked to the provider network, vlan, subnet resources.</li> <li>Based on this configuration item, the default vpc enable_eip_snat function can be docked to the existing vlan, subnet resources, while supporting the ipam</li> <li>If only the default vpc's enable_eip_snat mode is used with the old pod annotaion based eip fip snat, then the following configuration is not required.</li> <li>Based on this configuration you can not use the default vpc enable_eip_snat process, only by corresponding to vlan, subnet process, can be compatible with only custom vpc use eip snat usage scenarios.</li> </ul> <p>The neutron ovn mode also has a certain static file configuration designation that is, for now, generally consistent.</p>"},{"location":"en/advance/ovn-eip-fip-snat/#11-create-the-underlay-public-network","title":"1.1 Create the underlay public network","text":"<pre><code># provider-network\uff0c vlan\uff0c subnet\n# cat 01-provider-network.yaml\napiVersion: kubeovn.io/v1\nkind: ProviderNetwork\nmetadata:\n  name: external204\nspec:\n  defaultInterface: vlan\n\n# cat 02-vlan.yaml\napiVersion: kubeovn.io/v1\nkind: Vlan\nmetadata:\n  name: vlan204\nspec:\n  id: 204\nprovider: external204\n\n# cat 03-vlan-subnet.yaml\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n  name: external204\nspec:\n  protocol: IPv4\n  cidrBlock: 10.5.204.0/24\n  gateway: 10.5.204.254\n  vlan: vlan204\n  excludeIps:\n  - 10.5.204.1..10.5.204.100\n</code></pre>"},{"location":"en/advance/ovn-eip-fip-snat/#12-default-vpc-enable-eip_snat","title":"1.2 Default vpc enable eip_snat","text":"<pre><code># Enable the default vpc and the above underlay public provider subnet interconnection\ncat 00-centralized-external-gw-no-ip.yaml\napiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: ovn-external-gw-config\n  namespace: kube-system\ndata:\n  enable-external-gw: \"true\"\nexternal-gw-nodes: \"pc-node-1,pc-node-2,pc-node-3\" type: \"centralized\"  external-gw-nic: \"vlan\"\nexternal-gw-addr: \"10.5.204.254/24\"\n</code></pre> <p>This feature currently supports the ability to create lrp type ovn eip resources without specifying the lrp ip and mac, which is already supported for automatic acquisition. If specified, it is equivalent to specifying the ip to create an ovn-eip of type lrp. Of course, you can also manually create the lrp type ovn eip in advance.</p>"},{"location":"en/advance/ovn-eip-fip-snat/#13-custom-vpc-enable-eip-snat-fip-function","title":"1.3 Custom vpc enable eip snat fip function","text":"<pre><code># cat 00-ns.yml\napiVersion: v1\nkind: Namespace\nmetadata:\n  name: vpc1\n\n# cat 01-vpc-ecmp-enable-external-bfd.yml\nkind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\n  name: vpc1\nspec:\n  namespaces:\n  - vpc1\n  enableExternal: true\n# vpc enableExternal will automatically create an lrp association to the public network specified above\n\n# cat 02-subnet.yml\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n  name: vpc1-subnet1\nspec:\n  cidrBlock: 192.168.0.0/24\n  default: false\ndisableGatewayCheck: false\ndisableInterConnection: true\nenableEcmp: true\ngatewayNode: \"\"\ngatewayType: distributed\n  #gatewayType: centralized\nnatOutgoing: false\nprivate: false\nprotocol: IPv4\n  provider: ovn\n  vpc: vpc1\n  namespaces:\n  - vpc1\n</code></pre> <p>After the above template is applied, you should see the following resources exist</p> <pre><code># k ko nbctl show vpc1\nrouter 87ad06fd-71d5-4ff8-a1f0-54fa3bba1a7f (vpc1)\nport vpc1-vpc1-subnet1\n        mac: \"00:00:00:ED:8E:C7\"\nnetworks: [\"192.168.0.1/24\"]\nport vpc1-external204\n        mac: \"00:00:00:EF:05:C7\"\nnetworks: [\"10.5.204.105/24\"]\ngateway chassis: [7cedd14f-265b-42e5-ac17-e03e7a1f2342 276baccb-fe9c-4476-b41d-05872a94976d fd9f140c-c45d-43db-a6c0-0d4f8ea298dd]\nnat 21d853b0-f7b4-40bd-9a53-31d2e2745739\n        external ip: \"10.5.204.115\"\nlogical ip: \"192.168.0.0/24\"\ntype: \"snat\"\n</code></pre> <pre><code># k ko nbctl lr-route-list vpc1\nIPv4 Routes\nRoute Table &lt;main&gt;:\n                0.0.0.0/0              10.5.204.254 dst-ip\n# The route currently supports automatic maintenance\n</code></pre>"},{"location":"en/advance/ovn-eip-fip-snat/#2-ovn-eip","title":"2. ovn-eip","text":"<p>This function is designed and used in the same way as iptables-eip, ovn-eip currently has three types</p> <ul> <li>nat: indicates ovn dnat, fip, and snat. These nat types are recorded in status</li> <li>lrp: indicates the resource used to connect a vpc to the public network</li> <li>lsp: In the ovn BFD-based ecmp static route scenario, an ovs internal port is provided on the gateway node as the next hop of the ecmp route</li> </ul> <pre><code>---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: eip-static\nspec:\n  externalSubnet: external204\n  type: nat\n\n# Dynamically allocate an eip resource that is reserved for fip dnat_and_snat scenarios\n</code></pre>"},{"location":"en/advance/ovn-eip-fip-snat/#21-create-an-fip-for-pod","title":"2.1 Create an fip for pod","text":"<pre><code># k get po -o wide -n vpc1 vpc-1-busybox01\nNAME              READY   STATUS    RESTARTS   AGE     IP            NODE\nvpc-1-busybox01   1/1     Running   0          3d15h   192.168.0.2   pc-node-2\n\n# k get ip vpc-1-busybox01.vpc1\nNAME                   V4IP          V6IP   MAC                 NODE        SUBNET\nvpc-1-busybox01.vpc1   192.168.0.2          00:00:00:0A:DD:27   pc-node-2   vpc1-subnet1\n\n---\n\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: eip-static\nspec:\n  externalSubnet: external204\n  type: nat\n\n---\nkind: OvnFip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: eip-static\nspec:\n  ovnEip: eip-static\n  ipName: vpc-1-busybox01.vpc1  # the name of the ip crd, which is unique\n</code></pre> <pre><code># k get ofip\nNAME          VPC    V4EIP          V4IP          READY   IPTYPE   IPNAME\neip-for-vip   vpc1   10.5.204.106   192.168.0.3   true    vip      test-fip-vip\neip-static    vpc1   10.5.204.101   192.168.0.2   true             vpc-1-busybox01.vpc1\n# k get ofip eip-static\nNAME         VPC    V4EIP          V4IP          READY   IPTYPE   IPNAME\neip-static   vpc1   10.5.204.101   192.168.0.2   true             vpc-1-busybox01.vpc1\n\n[root@pc-node-1 03-cust-vpc]# ping 10.5.204.101\nPING 10.5.204.101 (10.5.204.101) 56(84) bytes of data.\n64 bytes from 10.5.204.101: icmp_seq=2 ttl=62 time=1.21 ms\n64 bytes from 10.5.204.101: icmp_seq=3 ttl=62 time=0.624 ms\n64 bytes from 10.5.204.101: icmp_seq=4 ttl=62 time=0.368 ms\n^C\n--- 10.5.204.101 ping statistics ---\n4 packets transmitted, 3 received, 25% packet loss, time 3049ms\nrtt min/avg/max/mdev = 0.368/0.734/1.210/0.352 ms\n[root@pc-node-1 03-cust-vpc]#\n\n# pod &lt;--&gt; node ping is working\n</code></pre> <pre><code># The key resources that this public ip can pass include the following ovn nb resources\n\n# k ko nbctl show vpc1\nrouter 87ad06fd-71d5-4ff8-a1f0-54fa3bba1a7f (vpc1)\nport vpc1-vpc1-subnet1\n        mac: \"00:00:00:ED:8E:C7\"\nnetworks: [\"192.168.0.1/24\"]\nport vpc1-external204\n        mac: \"00:00:00:EF:05:C7\"\nnetworks: [\"10.5.204.105/24\"]\ngateway chassis: [7cedd14f-265b-42e5-ac17-e03e7a1f2342 276baccb-fe9c-4476-b41d-05872a94976d fd9f140c-c45d-43db-a6c0-0d4f8ea298dd]\nnat 813523e7-c68c-408f-bd8c-cba30cb2e4f4\n        external ip: \"10.5.204.101\"\nlogical ip: \"192.168.0.2\"\ntype: \"dnat_and_snat\"\n</code></pre>"},{"location":"en/advance/ovn-eip-fip-snat/#22-create-an-fip-for-vip","title":"2.2 Create an fip for vip","text":"<p>In order to facilitate the use of some vip scenarios, such as inside kubevirt VM, keepalived use vip, kube-vip use vip, etc. the vip need public network access.</p> <pre><code># First create vip, eip, then bind eip to vip\n# cat vip.yaml\napiVersion: kubeovn.io/v1\nkind: Vip\nmetadata:\n  name: test-fip-vip\nspec:\n  subnet: vpc1-subnet1\n\n# cat 04-fip.yaml\n---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: eip-for-vip\nspec:\n  externalSubnet: external204\n  type: nat\n\n---\nkind: OvnFip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: eip-for-vip\nspec:\n  ovnEip: eip-for-vip\n  ipType: vip         # By default fip is for pod ip, here you need to specify the docking to vip resources\nipName: test-fip-vip\n</code></pre> <pre><code># k get ofip\nNAME          VPC    V4EIP          V4IP          READY   IPTYPE   IPNAME\neip-for-vip   vpc1   10.5.204.106   192.168.0.3   true    vip      test-fip-vip\n\n\n[root@pc-node-1 fip-vip]# ping  10.5.204.106\nPING 10.5.204.106 (10.5.204.106) 56(84) bytes of data.\n64 bytes from 10.5.204.106: icmp_seq=1 ttl=62 time=0.694 ms\n64 bytes from 10.5.204.106: icmp_seq=2 ttl=62 time=0.436 ms\n\n# node &lt;--&gt; pod fip is working\n\n# The way ip is used inside the pod is roughly as follows\n\n[root@pc-node-1 fip-vip]# k -n vpc1 exec -it vpc-1-busybox03 -- bash\n[root@vpc-1-busybox03 /]#\n[root@vpc-1-busybox03 /]#\n[root@vpc-1-busybox03 /]# ip a\n1: lo: &lt;LOOPBACK,UP,LOWER_UP&gt; mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000\nlink/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00\n    inet 127.0.0.1/8 scope host lo\n       valid_lft forever preferred_lft forever\n    inet6 ::1/128 scope host\n       valid_lft forever preferred_lft forever\n1568: eth0@if1569: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1500 qdisc noqueue state UP group default\n    link/ether 00:00:00:56:40:e5 brd ff:ff:ff:ff:ff:ff link-netnsid 0\ninet 192.168.0.5/24 brd 192.168.0.255 scope global eth0\n       valid_lft forever preferred_lft forever\n    inet 192.168.0.3/24 scope global secondary eth0  # vip here\nvalid_lft forever preferred_lft forever\n    inet6 fe80::200:ff:fe56:40e5/64 scope link\n       valid_lft forever preferred_lft forever\n\n[root@vpc-1-busybox03 /]# tcpdump -i eth0 host  192.168.0.3 -netvv\ntcpdump: listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes\n00:00:00:ed:8e:c7 &gt; 00:00:00:56:40:e5, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 62, id 44830, offset 0, flags [DF], proto ICMP (1), length 84)\n10.5.32.51 &gt; 192.168.0.3: ICMP echo request, id 177, seq 1, length 64\n00:00:00:56:40:e5 &gt; 00:00:00:ed:8e:c7, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 43962, offset 0, flags [none], proto ICMP (1), length 84)\n192.168.0.3 &gt; 10.5.32.51: ICMP echo reply, id 177, seq 1, length 64\n\n# pod internal can catch fip related icmp packets\n</code></pre>"},{"location":"en/advance/ovn-eip-fip-snat/#31-ovn-snat-corresponds-to-the-cidr-of-a-subnet","title":"3.1 ovn-snat corresponds to the CIDR of a subnet","text":"<p>This feature is designed and used in much the same way as iptables-snat</p> <pre><code># cat 03-subnet-snat.yaml\n---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: snat-for-subnet-in-vpc\nspec:\n  externalSubnet: external204\n  type: nat\n\n---\nkind: OvnSnatRule\napiVersion: kubeovn.io/v1\nmetadata:\n  name: snat-for-subnet-in-vpc\nspec:\n  ovnEip: snat-for-subnet-in-vpc\n  vpcSubnet: vpc1-subnet1 # eip corresponds to the entire network segment\n</code></pre>"},{"location":"en/advance/ovn-eip-fip-snat/#32-ovn-snat-corresponds-to-a-pod-ip","title":"3.2 ovn-snat corresponds to a pod IP","text":"<p>This feature is designed and used in much the same way as iptables-snat</p> <pre><code># cat 03-pod-snat.yaml\n---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\n  name: snat-for-pod-vpc-ip\nspec:\n  externalSubnet: external204\n  type: nat\n\n---\nkind: OvnSnatRule\napiVersion: kubeovn.io/v1\nmetadata:\n  name: snat01\nspec:\n  ovnEip: snat-for-pod-vpc-ip\n  ipName: vpc-1-busybox02.vpc1 # eip corresponds to a single pod ip\n</code></pre> <p>After the above resources are created, you can see the following resources that the snat public network feature depends on.</p> <pre><code># kubectl ko nbctl show vpc1\nrouter 87ad06fd-71d5-4ff8-a1f0-54fa3bba1a7f (vpc1)\nport vpc1-vpc1-subnet1\n        mac: \"00:00:00:ED:8E:C7\"\nnetworks: [\"192.168.0.1/24\"]\nport vpc1-external204\n        mac: \"00:00:00:EF:05:C7\"\nnetworks: [\"10.5.204.105/24\"]\ngateway chassis: [7cedd14f-265b-42e5-ac17-e03e7a1f2342 276baccb-fe9c-4476-b41d-05872a94976d fd9f140c-c45d-43db-a6c0-0d4f8ea298dd]\nnat 21d853b0-f7b4-40bd-9a53-31d2e2745739\n        external ip: \"10.5.204.115\"\nlogical ip: \"192.168.0.0/24\"\ntype: \"snat\"\nnat da77a11f-c523-439c-b1d1-72c664196a0f\n        external ip: \"10.5.204.116\"\nlogical ip: \"192.168.0.4\"\ntype: \"snat\"\n</code></pre> <pre><code>[root@pc-node-1 03-cust-vpc]# k get po -A -o wide  | grep busy\nvpc1            vpc-1-busybox01                                 1/1     Running   0                3d15h   192.168.0.2   pc-node-2   &lt;none&gt;           &lt;none&gt;\nvpc1            vpc-1-busybox02                                 1/1     Running   0                17h     192.168.0.4   pc-node-1   &lt;none&gt;           &lt;none&gt;\nvpc1            vpc-1-busybox03                                 1/1     Running   0                17h     192.168.0.5   pc-node-1   &lt;none&gt;           &lt;none&gt;\nvpc1            vpc-1-busybox04                                 1/1     Running   0                17h     192.168.0.6   pc-node-3   &lt;none&gt;           &lt;none&gt;\nvpc1            vpc-1-busybox05                                 1/1     Running   0                17h     192.168.0.7   pc-node-1   &lt;none&gt;           &lt;none&gt;\n\n# k exec -it -n vpc1            vpc-1-busybox04   bash\nkubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.\n[root@vpc-1-busybox04 /]#\n[root@vpc-1-busybox04 /]#\n[root@vpc-1-busybox04 /]# ip a\n1: lo: &lt;LOOPBACK,UP,LOWER_UP&gt; mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000\nlink/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00\n    inet 127.0.0.1/8 scope host lo\n       valid_lft forever preferred_lft forever\n    inet6 ::1/128 scope host\n       valid_lft forever preferred_lft forever\n17095: eth0@if17096: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1500 qdisc noqueue state UP group default\n    link/ether 00:00:00:76:94:55 brd ff:ff:ff:ff:ff:ff link-netnsid 0\ninet 192.168.0.6/24 brd 192.168.0.255 scope global eth0\n       valid_lft forever preferred_lft forever\n    inet6 fe80::200:ff:fe76:9455/64 scope link\n       valid_lft forever preferred_lft forever\n[root@vpc-1-busybox04 /]# ping 223.5.5.5\nPING 223.5.5.5 (223.5.5.5) 56(84) bytes of data.\n64 bytes from 223.5.5.5: icmp_seq=1 ttl=114 time=22.2 ms\n64 bytes from 223.5.5.5: icmp_seq=2 ttl=114 time=21.8 ms\n\n[root@pc-node-1 03-cust-vpc]# k exec -it -n vpc1            vpc-1-busybox02   bash\nkubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.\n[root@vpc-1-busybox02 /]#\n[root@vpc-1-busybox02 /]#\n[root@vpc-1-busybox02 /]#\n[root@vpc-1-busybox02 /]# ip a\n1: lo: &lt;LOOPBACK,UP,LOWER_UP&gt; mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000\nlink/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00\n    inet 127.0.0.1/8 scope host lo\n       valid_lft forever preferred_lft forever\n    inet6 ::1/128 scope host\n       valid_lft forever preferred_lft forever\n1566: eth0@if1567: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1500 qdisc noqueue state UP group default\n    link/ether 00:00:00:0b:e9:d0 brd ff:ff:ff:ff:ff:ff link-netnsid 0\ninet 192.168.0.4/24 brd 192.168.0.255 scope global eth0\n       valid_lft forever preferred_lft forever\n    inet6 fe80::200:ff:fe0b:e9d0/64 scope link\n       valid_lft forever preferred_lft forever\n[root@vpc-1-busybox02 /]# ping 223.5.5.5\nPING 223.5.5.5 (223.5.5.5) 56(84) bytes of data.\n64 bytes from 223.5.5.5: icmp_seq=2 ttl=114 time=22.7 ms\n64 bytes from 223.5.5.5: icmp_seq=3 ttl=114 time=22.6 ms\n64 bytes from 223.5.5.5: icmp_seq=4 ttl=114 time=22.1 ms\n^C\n--- 223.5.5.5 ping statistics ---\n4 packets transmitted, 3 received, 25% packet loss, time 3064ms\nrtt min/avg/max/mdev = 22.126/22.518/22.741/0.278 ms\n\n# the two pods can access the external network based on these two type snat resources respectively\n</code></pre>"},{"location":"en/advance/ovn-eip-fip-snat/#41-ovn-dnat-binds-a-dnat-to-a-pod","title":"4.1 ovn-dnat binds a DNAT to a pod","text":"<pre><code>kind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-static\nspec:\nexternalSubnet: underlay\ntype: nat\n---\nkind: OvnDnatRule\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-dnat\nspec:\novnEip: eip-dnat\nipName: vpc-1-busybox01.vpc1 # Note that this is the name of the pod IP CRD and it is unique\nprotocol: tcp\ninternalPort: \"22\"\nexternalPort: \"22\"\n</code></pre> <p>The configuration of OvnDnatRule is similar to that of IptablesDnatRule.</p> <pre><code># kubectl get oeip eip-dnat\nNAME       V4IP        V6IP   MAC                 TYPE   READY\neip-dnat   10.5.49.4          00:00:00:4D:CE:49   dnat   true\n\n# kubectl get odnat\nNAME                   EIP                    PROTOCOL   V4EIP        V4IP           INTERNALPORT   EXTERNALPORT   IPNAME                                READY\neip-dnat               eip-dnat               tcp        10.5.49.4    192.168.0.3    22             22             vpc-1-busybox01.vpc1                  true\n</code></pre>"},{"location":"en/advance/ovn-eip-fip-snat/#42-ovn-dnat-binds-a-dnat-to-a-vip","title":"4.2 ovn-dnat binds a DNAT to a VIP","text":"<pre><code>kind: OvnDnatRule\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-dnat\nspec:\nipType: vip  # By default, Dnat is oriented towards pod IPs. Here, it is necessary to specify that it is connected to VIP resources\novnEip: eip-dnat\nipName: test-dnat-vip\nprotocol: tcp\ninternalPort: \"22\"\nexternalPort: \"22\"\n</code></pre> <p>The configuration of OvnDnatRule is similar to that of IptablesDnatRule.</p> <pre><code># kubectl get vip test-dnat-vip\nNAME            V4IP          PV4IP   MAC                 PMAC   V6IP   PV6IP   SUBNET         READY\ntest-dnat-vip   192.168.0.4           00:00:00:D0:C0:B5                         vpc1-subnet1   true\n\n# kubectl get oeip eip-dnat\nNAME       V4IP        V6IP   MAC                 TYPE   READY\neip-dnat   10.5.49.4          00:00:00:4D:CE:49   dnat   true\n\n# kubectl get odnat eip-dnat \nNAME       EIP        PROTOCOL   V4EIP       V4IP          INTERNALPORT   EXTERNALPORT   IPNAME          READY\neip-dnat   eip-dnat   tcp        10.5.49.4   192.168.0.4   22             22             test-dnat-vip   true\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/ovn-ipsec/","title":"Encrypt inter-node communication using IPsec","text":"<p>This function is supported after v1.10.11 and v1.11.4, and the kernel version is at least 3.10.0 or above.</p>"},{"location":"en/advance/ovn-ipsec/#start-ipsec","title":"start IPsec","text":"<p>Copy the script from the Kube-OVN source code ipsec.sh, execute the command as follows, the script will call ovs-pki to generate and distribute the certificate required for encryption:</p> <pre><code>bash ipsec.sh init\n</code></pre> <p>After the execution is completed, the nodes will negotiate for a period of time to establish an IPsec tunnel. The experience value is between ten seconds and one minute.You can check the IPsec status with the following command:</p> <pre><code># bash ipsec.sh status\nPod {ovs-ovn-d7hdt} ipsec status...\nInterface name: ovn-a4718e-0 v1 (CONFIGURED)\nTunnel Type:    geneve\n  Local IP:       172.18.0.2\n  Remote IP:      172.18.0.4\n  Address Family: IPv4\n  SKB mark:       None\n  Local cert:     /etc/ipsec.d/certs/8aebd9df-46ef-47b9-85e3-73e9a765296d-cert.pem\n  Local name:     8aebd9df-46ef-47b9-85e3-73e9a765296d\n  Local key:      /etc/ipsec.d/private/8aebd9df-46ef-47b9-85e3-73e9a765296d-privkey.pem\n  Remote cert:    None\n  Remote name:    a4718e55-5b85-4f46-90e6-63527d080590\n  CA cert:        /etc/ipsec.d/cacerts/cacert.pem\n  PSK:            None\n  Custom Options: {}\nOfport:         2\nCFM state:      Disabled\nKernel policies installed:\n  src 172.18.0.2/32 dst 172.18.0.4/32 proto udp sport 6081\nsrc 172.18.0.2/32 dst 172.18.0.4/32 proto udp sport 6081\nsrc 172.18.0.2/32 dst 172.18.0.4/32 proto udp dport 6081\nsrc 172.18.0.2/32 dst 172.18.0.4/32 proto udp dport 6081\nKernel security associations installed:\n  sel src 172.18.0.2/32 dst 172.18.0.4/32 proto udp sport 6081\nsel src 172.18.0.4/32 dst 172.18.0.2/32 proto udp dport 6081\nsel src 172.18.0.2/32 dst 172.18.0.4/32 proto udp dport 6081\nsel src 172.18.0.4/32 dst 172.18.0.2/32 proto udp sport 6081\nIPsec connections that are active:\n\n Pod {ovs-ovn-fvbbj} ipsec status...\nInterface name: ovn-8aebd9-0 v1 (CONFIGURED)\nTunnel Type:    geneve\n  Local IP:       172.18.0.4\n  Remote IP:      172.18.0.2\n  Address Family: IPv4\n  SKB mark:       None\n  Local cert:     /etc/ipsec.d/certs/a4718e55-5b85-4f46-90e6-63527d080590-cert.pem\n  Local name:     a4718e55-5b85-4f46-90e6-63527d080590\n  Local key:      /etc/ipsec.d/private/a4718e55-5b85-4f46-90e6-63527d080590-privkey.pem\n  Remote cert:    None\n  Remote name:    8aebd9df-46ef-47b9-85e3-73e9a765296d\n  CA cert:        /etc/ipsec.d/cacerts/cacert.pem\n  PSK:            None\n  Custom Options: {}\nOfport:         1\nCFM state:      Disabled\nKernel policies installed:\n  src 172.18.0.4/32 dst 172.18.0.2/32 proto udp dport 6081\nsrc 172.18.0.4/32 dst 172.18.0.2/32 proto udp dport 6081\nsrc 172.18.0.4/32 dst 172.18.0.2/32 proto udp sport 6081\nsrc 172.18.0.4/32 dst 172.18.0.2/32 proto udp sport 6081\nKernel security associations installed:\n  sel src 172.18.0.4/32 dst 172.18.0.2/32 proto udp dport 6081\nsel src 172.18.0.2/32 dst 172.18.0.4/32 proto udp sport 6081\nsel src 172.18.0.4/32 dst 172.18.0.2/32 proto udp sport 6081\nsel src 172.18.0.2/32 dst 172.18.0.4/32 proto udp dport 6081\nIPsec connections that are active:\n</code></pre> <p>After the establishment is complete, you can capture packets and observe that the packets have been encrypted:</p> <pre><code># tcpdump -i eth0 -nel esp\n10:01:40.349896 IP kube-ovn-worker &gt; kube-ovn-control-plane.kind: ESP(spi=0xcc91322a,seq=0x13d0), length 156\n10:01:40.350015 IP kube-ovn-control-plane.kind &gt; kube-ovn-worker: ESP(spi=0xc8df4221,seq=0x1d37), length 156\n</code></pre> <p>After executing the script, you can turn off IPsec by executing the command:</p> <pre><code># bash ipsec.sh stop\n</code></pre> <p>Or execute the command to open it again:</p> <pre><code># bash ipsec.sh start\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/ovn-l3-ha-based-ecmp-with-bfd/","title":"Support OVN SNAT L3 HA Based ECMP and BFD Static Route","text":"<p>Custom vpc based on ovn snat after ecmp based static route hash to multiple gw node ovnext0 NICs out of the public network</p> <ul> <li>Supports bfd-based high availability</li> <li>Only supports hash load balancing</li> </ul> <pre><code>graph LR\n\npod--&gt;vpc-subnet--&gt;vpc--&gt;snat--&gt;ecmp--&gt;external-subnet--&gt;gw-node1-ovnext0--&gt; node1-external-switch\nexternal-subnet--&gt;gw-node2-ovnext0--&gt; node2-external-switch\nexternal-subnet--&gt;gw-node3-ovnext0--&gt; node3-external-switch</code></pre> <p>This functions basically the same as ovn-eip-fip-snat.md .</p> <p>As for the different parts, which will be specified in the following sections, mainly including the creation of ovn-eip of lsp type and the automatic maintenance of bfd as well as ecmp static routes based on vpc enable_bfd.</p>"},{"location":"en/advance/ovn-l3-ha-based-ecmp-with-bfd/#1-deployment","title":"1. Deployment","text":""},{"location":"en/advance/ovn-l3-ha-based-ecmp-with-bfd/#11-create-the-underlay-public-network","title":"1.1 Create the underlay public network","text":""},{"location":"en/advance/ovn-l3-ha-based-ecmp-with-bfd/#12-default-vpc-enable-eip_snat","title":"1.2 Default vpc enable eip_snat","text":""},{"location":"en/advance/ovn-l3-ha-based-ecmp-with-bfd/#13-custom-vpc-enable-eip-snat-fip-function","title":"1.3 Custom vpc enable eip snat fip function","text":"<p>The above section is exactly the same with ovn-eip-fip-snat.md.</p> <p>After these functions are verified, the vpc can be switched directly to the ecmp-based bfd static route based on the following way, or of course, switched directly back.</p> <p>Before customizing vpc to use this feature, you need to provide some gateway nodes, at least 2. Note that the name of the current implementation of ovn-eip must be consistent with the gateway node name, no automated maintenance is currently done for this resource.</p> <pre><code># cat gw-node-eip.yaml\n---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\nname: pc-node-1\nspec:\nexternalSubnet: external204\ntype: lsp\n\n---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\nname: pc-node-2\nspec:\nexternalSubnet: external204\ntype: lsp\n\n---\nkind: OvnEip\napiVersion: kubeovn.io/v1\nmetadata:\nname: pc-node-3\nspec:\nexternalSubnet: external204\ntype: lsp\n</code></pre> <p>Since this scenario is currently designed for vpc ecmp out of the public network, the gateway node above will not trigger the creation of a gateway NIC when there is no vpc enabled bfd, i.e. when there is no ovn eip (lrp) with enable bfd labeled, and will not be able to successfully start listening to the bfd session on the other side.</p>"},{"location":"en/advance/ovn-l3-ha-based-ecmp-with-bfd/#2-custom-vpc-enable-ecmp-bfd-l3-ha-public-network-function","title":"2. Custom vpc enable ecmp bfd L3 HA public network function","text":"<pre><code># cat 01-vpc-ecmp-enable-external-bfd.yml\nkind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\n  name: vpc1\nspec:\n  namespaces:\n  - vpc1\n  enableExternal: true\nenableBfd: true # bfd switch can be switched at will\n#enableBfd: false \n\n\n# cat 02-subnet.yml\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n  name: vpc1-subnet1\nspec:\n  cidrBlock: 192.168.0.0/24\n  default: false\ndisableGatewayCheck: false\ndisableInterConnection: true\nenableEcmp: true  # enable ecmp\ngatewayNode: \"\"\ngatewayType: distributed\n  #gatewayType: centralized\nnatOutgoing: false\nprivate: false\nprotocol: IPv4\n  provider: ovn\n  vpc: vpc1\n  namespaces:\n  - vpc1\n</code></pre> <p>note:</p> <ol> <li>Customize ecmp under vpc to use only static ecmp bfd routes. vpc enableBfd and subnet enableEcmp will only take effect if they are enabled at the same time, before static ecmp bfd routes are automatically managed.</li> <li>If the above configuration is turned off, it will automatically switch back to the regular default static route.</li> <li>This feature is not available for the default vpc, only custom vpc is supported, the default vpc has more complex policy routing.</li> <li>The enableEcmp of the subnet of the custom vpc uses only static routes, the gateway type gatewayType has no effect.</li> <li>When EnableExternal is turned off in vpc, the external network cannot be passed inside vpc.</li> <li>When EnableExternal is enabled on vpc, when EnableBfd is turned off, it will be based on the normal default route to the external network and will not have high availability.</li> </ol> <pre><code># After the above template is applied the ovn logic layer should see the following resources\n# k get vpc\nNAME          ENABLEEXTERNAL   ENABLEBFD   STANDBY   SUBNETS                                NAMESPACES\novn-cluster   true                         true      [\"external204\",\"join\",\"ovn-default\"]\nvpc1          true             true        true      [\"vpc1-subnet1\"]                       [\"vpc1\"]\n\n# Default vpc does not support ENABLEBFD\n# Custom vpc is supported and enabled\n\n# 1. bfd table created\n# k ko nbctl list bfd\n_uuid               : be7df545-2c4c-4751-878f-b3507987f050\ndetect_mult         : 3\ndst_ip              : \"10.5.204.121\"\nexternal_ids        : {}\nlogical_port        : vpc1-external204\nmin_rx              : 100\nmin_tx              : 100\noptions             : {}\nstatus              : up\n\n_uuid               : 684c4489-5b59-4693-8d8c-3beab93f8093\ndetect_mult         : 3\ndst_ip              : \"10.5.204.109\"\nexternal_ids        : {}\nlogical_port        : vpc1-external204\nmin_rx              : 100\nmin_tx              : 100\noptions             : {}\nstatus              : up\n\n_uuid               : f0f62077-2ae9-4e79-b4f8-a446ec6e784c\ndetect_mult         : 3\ndst_ip              : \"10.5.204.108\"\nexternal_ids        : {}\nlogical_port        : vpc1-external204\nmin_rx              : 100\nmin_tx              : 100\noptions             : {}\nstatus              : up\n\n### Note that all statuses should normally be up\n\n# 2. bfd ecmp static routes table created\n# k ko nbctl lr-route-list vpc1\nIPv4 Routes\nRoute Table &lt;main&gt;:\n           192.168.0.0/24              10.5.204.108 src-ip ecmp ecmp-symmetric-reply bfd\n           192.168.0.0/24              10.5.204.109 src-ip ecmp ecmp-symmetric-reply bfd\n           192.168.0.0/24              10.5.204.121 src-ip ecmp ecmp-symmetric-reply bfd\n\n# 3. Static Route Details\n# k ko nbctl find Logical_Router_Static_Route  policy=src-ip options=ecmp_symmetric_reply=\"true\"\n_uuid               : 3aacb384-d5ee-4b14-aebf-59e8c11717ba\nbfd                 : 684c4489-5b59-4693-8d8c-3beab93f8093\nexternal_ids        : {}\nip_prefix           : \"192.168.0.0/24\"\nnexthop             : \"10.5.204.109\"\noptions             : {ecmp_symmetric_reply=\"true\"}\noutput_port         : []\npolicy              : src-ip\nroute_table         : \"\"\n\n_uuid               : 18bcc585-bc05-430b-925b-ef673c8e1aef\nbfd                 : f0f62077-2ae9-4e79-b4f8-a446ec6e784c\nexternal_ids        : {}\nip_prefix           : \"192.168.0.0/24\"\nnexthop             : \"10.5.204.108\"\noptions             : {ecmp_symmetric_reply=\"true\"}\noutput_port         : []\npolicy              : src-ip\nroute_table         : \"\"\n\n_uuid               : 7d0a4e6b-cde0-4110-8176-fbaf19738498\nbfd                 : be7df545-2c4c-4751-878f-b3507987f050\nexternal_ids        : {}\nip_prefix           : \"192.168.0.0/24\"\nnexthop             : \"10.5.204.121\"\noptions             : {ecmp_symmetric_reply=\"true\"}\noutput_port         : []\npolicy              : src-ip\nroute_table         : \"\"\n</code></pre> <pre><code># Also, the following resources should be available at all gateway nodes\n\n[root@pc-node-1 ~]# ip netns exec ovnext bash ip a\n/usr/sbin/ip: /usr/sbin/ip: cannot execute binary file\n[root@pc-node-1 ~]#\n[root@pc-node-1 ~]# ip netns exec ovnext ip a\n1: lo: &lt;LOOPBACK,UP,LOWER_UP&gt; mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000\nlink/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00\n    inet 127.0.0.1/8 scope host lo\n       valid_lft forever preferred_lft forever\n    inet6 ::1/128 scope host\n       valid_lft forever preferred_lft forever\n1541: ovnext0: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1400 qdisc noqueue state UNKNOWN group default qlen 1000\nlink/ether 00:00:00:ab:bd:87 brd ff:ff:ff:ff:ff:ff\n    inet 10.5.204.108/24 brd 10.5.204.255 scope global ovnext0\n       valid_lft forever preferred_lft forever\n    inet6 fe80::200:ff:feab:bd87/64 scope link\n       valid_lft forever preferred_lft forever\n[root@pc-node-1 ~]#\n[root@pc-node-1 ~]# ip netns exec ovnext route -n\nKernel IP routing table\nDestination     Gateway         Genmask         Flags Metric Ref    Use Iface\n0.0.0.0         10.5.204.254    0.0.0.0         UG    0      0        0 ovnext0\n10.5.204.0      0.0.0.0         255.255.255.0   U     0      0        0 ovnext0\n\n\n[root@pc-node-1 ~]# ip netns exec ovnext bfdd-control status\nThere are 1 sessions:\nSession 1\nid=1 local=10.5.204.108 (p) remote=10.5.204.122 state=Up\n\n## This is the other end of the lrp bfd session and one of the next hops of the lrp ecmp\n\n\n[root@pc-node-1 ~]# ip netns exec ovnext ping -c1 223.5.5.5\nPING 223.5.5.5 (223.5.5.5) 56(84) bytes of data.\n64 bytes from 223.5.5.5: icmp_seq=1 ttl=115 time=21.6 ms\n\n# No problem to the public network\n</code></pre> <p>catch outgoing packets within the ovnext ns of a gateway node</p> <pre><code># tcpdump -i ovnext0 host 223.5.5.5 -netvv\ndropped privs to tcpdump\ntcpdump: listening on ovnext0, link-type EN10MB (Ethernet), capture size 262144 bytes\n^C\n0 packets captured\n0 packets received by filter\n0 packets dropped by kernel\n[root@pc-node-1 ~]# exit\n[root@pc-node-1 ~]# ssh pc-node-2\nLast login: Thu Feb 23 09:21:08 2023 from 10.5.32.51\n[root@pc-node-2 ~]# ip netns exec ovnext bash\n[root@pc-node-2 ~]# tcpdump -i ovnext0 host 223.5.5.5 -netvv\ndropped privs to tcpdump\ntcpdump: listening on ovnext0, link-type EN10MB (Ethernet), capture size 262144 bytes\n^C\n0 packets captured\n0 packets received by filter\n0 packets dropped by kernel\n[root@pc-node-2 ~]# exit\n[root@pc-node-2 ~]# logout\nConnection to pc-node-2 closed.\n[root@pc-node-1 ~]# ssh pc-node-3\nLast login: Thu Feb 23 08:32:41 2023 from 10.5.32.51\n[root@pc-node-3 ~]#  ip netns exec ovnext bash\n[root@pc-node-3 ~]# tcpdump -i ovnext0 host 223.5.5.5 -netvv\ndropped privs to tcpdump\ntcpdump: listening on ovnext0, link-type EN10MB (Ethernet), capture size 262144 bytes\n00:00:00:2d:f8:ce &gt; 00:00:00:fd:b2:a4, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 57978, offset 0, flags [DF], proto ICMP (1), length 84)\n10.5.204.102 &gt; 223.5.5.5: ICMP echo request, id 22, seq 71, length 64\n00:00:00:fd:b2:a4 &gt; dc:ef:80:5a:44:1a, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 62, id 57978, offset 0, flags [DF], proto ICMP (1), length 84)\n10.5.204.102 &gt; 223.5.5.5: ICMP echo request, id 22, seq 71, length 64\n^C\n2 packets captured\n2 packets received by filter\n0 packets dropped by kernel\n[root@pc-node-3 ~]#\n</code></pre>"},{"location":"en/advance/ovn-l3-ha-based-ecmp-with-bfd/#3-turn-off-bfd-mode","title":"3. Turn off bfd mode","text":"<p>In some scenarios, you may want to use a (centralized) single gateway directly out of the public network, which is the same as the default vpc enable_eip_snat usage pattern</p> <pre><code># cat 01-vpc-ecmp-enable-external-bfd.yml\nkind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\n  name: vpc2\nspec:\n  namespaces:\n  - vpc2\n  enableExternal: true\n#enableBfd: true\nenableBfd: false\n\n## set it false add apply\n\n# k ko nbctl lr-route-list vpc2\nIPv4 Routes\nRoute Table &lt;main&gt;:\n                0.0.0.0/0              10.5.204.254 dst-ip\n\n# After application the route will switch back to the normal default static route\n# nbctl list bfd, the bfd session associated with lrp has been removed\n# And the opposite side of the bfd session in ovnext ns is automatically removed\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/ovn-remote-port-mirroring/","title":"OVN Remote Port Mirroring","text":"<p>This feature provides ability to mirror the traffic of the specified Pod and direction, and to send the mirrored traffic to a remote destination.</p> <p>This feature requires Kube-OVN version not lower than v1.12.</p>"},{"location":"en/advance/ovn-remote-port-mirroring/#install-multus-cni","title":"Install Multus-CNI","text":"<p>Install Multus-CNI by referring the Multus-CNI Document.</p>"},{"location":"en/advance/ovn-remote-port-mirroring/#create-networkattachmentdefinition","title":"Create NetworkAttachmentDefinition","text":"<p>Create the following NetworkAttachmentDefinition:</p> <pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: attachnet\nnamespace: default\nspec:\nconfig: |\n{\n\"cniVersion\": \"0.3.1\",\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"attachnet.default.ovn\"\n}\n</code></pre> <p>Format of the <code>provider</code> field is <code>&lt;NAME&gt;.&lt;NAMESPACE&gt;.ovn</code>.</p>"},{"location":"en/advance/ovn-remote-port-mirroring/#create-underlay-network","title":"Create Underlay Network","text":"<p>The mirrored traffic is encapsulated before transmition, so MTU of the network used to transmit the traffic should be greater than the mirrored LSP/Pod. Here we are using an underlay network.</p> <p>Create the following underlay network:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: ProviderNetwork\nmetadata:\nname: net1\nspec:\ndefaultInterface: eth1\n---\napiVersion: kubeovn.io/v1\nkind: Vlan\nmetadata:\nname: vlan1\nspec:\nid: 0\nprovider: net1\n---\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: subnet1\nspec:\nprotocol: IPv4\ncidrBlock: 172.19.0.0/16\nexcludeIps:\n- 172.19.0.2..172.19.0.20\ngateway: 172.19.0.1\nvlan: vlan1\nprovider: attachnet.default.ovn\n</code></pre> <p>The subnet's <code>provider</code> MUST be the same as the <code>provider</code> of the NetworkAttachmentDefinition created above.</p>"},{"location":"en/advance/ovn-remote-port-mirroring/#create-receiving-pod","title":"Create Receiving Pod","text":"<p>Create the following Pod\uff1a</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: pod1\nannotations:\nk8s.v1.cni.cncf.io/networks: default/attachnet\nspec:\ncontainers:\n- name: bash\nimage: docker.io/kubeovn/kube-ovn:v1.13.0\nargs:\n- bash\n- -c\n- sleep infinity\nsecurityContext:\nprivileged: true\n</code></pre> <p>After the Pod has been created, checkout the IP addresses:</p> <pre><code>$ kubectl get ips | grep pod1\npod1.default                        10.16.0.12   00:00:00:FF:34:24  kube-ovn-worker  ovn-default\npod1.default.attachnet.default.ovn  172.19.0.21  00:00:00:A0:30:68  kube-ovn-worker  subnet1\n</code></pre> <p>The IP address <code>172.19.0.21</code> will be used later.</p>"},{"location":"en/advance/ovn-remote-port-mirroring/#create-ovn-remote-port-mirroring","title":"Create OVN Remote Port Mirroring","text":"<p>Create the following OVN remote port mirroring\uff1a</p> <pre><code>kubectl ko nbctl mirror-add mirror1 gre 99 from-lport 172.19.0.21\nkubectl ko nbctl lsp-attach-mirror coredns-787d4945fb-gpnkb.kube-system mirror1\n</code></pre> <p><code>coredns-787d4945fb-gpnkb.kube-system</code> is the OVN LSP name with a format <code>&lt;POD_NAME&gt;.&lt;POD_NAMESPACE&gt;</code>.</p> <p>Here is the OVN command usage:</p> <pre><code>ovn-nbctl mirror-add &lt;NAME&gt; &lt;TYPE&gt; &lt;INDEX&gt; &lt;FILTER&gt; &lt;IP&gt;\n\nNAME   - add a mirror with given name\nTYPE   - specify TYPE 'gre' or 'erspan'\nINDEX  - specify the tunnel INDEX value\n         (indicates key if GRE, erpsan_idx if ERSPAN)\nFILTER - specify FILTER for mirroring selection\n         ('to-lport' / 'from-lport')\nIP     - specify Sink / Destination i.e. Remote IP\n\novn-nbctl mirror-del [NAME]         remove mirrors\novn-nbctl mirror-list               print mirrors\n\novn-nbctl lsp-attach-mirror PORT MIRROR   attach source PORT to MIRROR\novn-nbctl lsp-detach-mirror PORT MIRROR   detach source PORT from MIRROR\n</code></pre>"},{"location":"en/advance/ovn-remote-port-mirroring/#configure-receiving-pod","title":"Configure Receiving Pod","text":"<p>Execute the following commands in the Pod:</p> <pre><code>root@pod1:/kube-ovn# ip link add mirror1 type gretap local 172.19.0.21 key 99 dev net1\nroot@pod1:/kube-ovn# ip link set mirror1 up\n</code></pre> <p>Now you can capture the mirrored packets:</p> <pre><code>root@pod1:/kube-ovn# tcpdump -i mirror1 -nnve\ntcpdump: listening on mirror1, link-type EN10MB (Ethernet), snapshot length 262144 bytes\n05:13:30.328808 00:00:00:a3:f5:e2 &gt; 00:00:00:97:0f:6e, ethertype ARP (0x0806), length 42: Ethernet (len 6), IPv4 (len 4), Request who-has 10.16.0.7 tell 10.16.0.4, length 28\n05:13:30.559167 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 212: (tos 0x0, ttl 64, id 57364, offset 0, flags [DF], proto UDP (17), length 198)\n10.16.0.4.53 &gt; 10.16.0.6.50472: 34511 NXDomain*- 0/1/1 (170)\n05:13:30.559343 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 212: (tos 0x0, ttl 64, id 57365, offset 0, flags [DF], proto UDP (17), length 198)\n10.16.0.4.53 &gt; 10.16.0.6.45177: 1659 NXDomain*- 0/1/1 (170)\n05:13:30.560625 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 200: (tos 0x0, ttl 64, id 57367, offset 0, flags [DF], proto UDP (17), length 186)\n10.16.0.4.53 &gt; 10.16.0.6.43848: 2636*- 0/1/1 (158)\n05:13:30.562774 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 191: (tos 0x0, ttl 64, id 57368, offset 0, flags [DF], proto UDP (17), length 177)\n10.16.0.4.53 &gt; 10.16.0.6.37755: 48737 NXDomain*- 0/1/1 (149)\n05:13:30.563523 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 187: (tos 0x0, ttl 64, id 57369, offset 0, flags [DF], proto UDP (17), length 173)\n10.16.0.4.53 &gt; 10.16.0.6.53887: 45519 NXDomain*- 0/1/1 (145)\n05:13:30.564940 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 201: (tos 0x0, ttl 64, id 57370, offset 0, flags [DF], proto UDP (17), length 187)\n10.16.0.4.53 &gt; 10.16.0.6.40846: 25745 NXDomain*- 0/1/1 (159)\n05:13:30.565140 00:00:00:a3:f5:e2 &gt; 00:00:00:89:d5:cc, ethertype IPv4 (0x0800), length 201: (tos 0x0, ttl 64, id 57371, offset 0, flags [DF], proto UDP (17), length 187)\n10.16.0.4.53 &gt; 10.16.0.6.45214: 61875 NXDomain*- 0/1/1 (159)\n05:13:30.566023 00:00:00:a3:f5:e2 &gt; 00:00:00:55:e4:4e, ethertype IPv4 (0x0800), length 80: (tos 0x0, ttl 64, id 45937, offset 0, flags [DF], proto UDP (17), length 66)\n10.16.0.4.44116 &gt; 172.18.0.1.53: 16025+ [1au] AAAA? alauda.cn. (38)\n</code></pre>"},{"location":"en/advance/ovn-remote-port-mirroring/#notice","title":"Notice","text":"<ol> <li>If you are using ERSPAN as the encapsulation protocol, the Linux kernel version of the OVN nodes and remote devices must not be lower than 4.14. If you are using ERSPAN as the encapsulation protocol and using IPv6 as the transport network, the Linux kernel version must not be lower than 4.16.</li> <li>The transmission of mirrored traffic is unidirectional, so you only need to ensure that the OVN node can access the remote device.</li> </ol> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/performance-tuning/","title":"Performance Tuning","text":"<p>To keep the installation simple and feature-complete, the default installation script for Kube-OVN does not have performance-specific optimizations. If the applications are sensitive to latency and throughput, administrators can use this document to make specific performance optimizations.</p> <p>The community will continue to iterate on the performance. Some general performance optimizations have been integrated into the latest version, so it is recommended to use the latest version to get better default performance.</p> <p>For more on the process and methodology of performance optimization, please watch the video Kube-OVN \u5bb9\u5668\u6027\u80fd\u4f18\u5316\u4e4b\u65c5\u3002</p>"},{"location":"en/advance/performance-tuning/#benchmarking","title":"Benchmarking","text":"<p>Because the hardware and software environments vary greatly, the performance test data provided here can only be used as a reference, and the actual test results may differ significantly from the results in this document. It is recommended to compare the performance test results before and after optimization, and the performance comparison between the host network and the container network.</p>"},{"location":"en/advance/performance-tuning/#overlay-performance-comparison-before-and-after-optimization","title":"Overlay Performance Comparison before and after Optimization","text":"<p>Environment:</p> <ul> <li>Kubernetes: 1.22.0</li> <li>OS: CentOS 7</li> <li>Kube-OVN: 1.8.0 Overlay Mode</li> <li>CPU: Intel(R) Xeon(R) E-2278G</li> <li>Network: 2*10Gbps, xmit_hash_policy=layer3+4</li> </ul> <p>We use <code>qperf -t 60 &lt;server ip&gt; -ub -oo msg_size:1 -vu tcp_lat tcp_bw udp_lat udp_bw</code> to test bandwidth and latency of tcp/udp in 1-byte packets and the host network, respectively.</p> Type tcp_lat (us) udp_lat (us) tcp_bw (Mb/s) udp_bw(Mb/s) Kube-OVN Default 25.7 22.9 27.1 1.59 Kube-OVN Optimized 13.9 12.9 27.6 5.57 HOST Network 13.1 12.4 28.2 6.02"},{"location":"en/advance/performance-tuning/#overlay-underlay-and-calico-comparison","title":"Overlay\uff0c Underlay and Calico Comparison","text":"<p>Next, we compare the overlay and underlay performance of the optimized Kube-OVN at different packet sizes with Calico's <code>IPIP Always</code>, <code>IPIP never</code> and the host network.</p> <p>Environment:</p> <ul> <li>Kubernetes: 1.22.0</li> <li>OS: CentOS 7</li> <li>Kube-OVN: 1.8.0</li> <li>CPU: AMD EPYC 7402P 24-Core Processor</li> <li>Network: Intel Corporation Ethernet Controller XXV710 for 25GbE SFP28</li> </ul> <p><code>qperf -t 60 &lt;server ip&gt; -ub -oo msg_size:1 -vu tcp_lat tcp_bw udp_lat udp_bw</code></p> Type tcp_lat (us) udp_lat (us) tcp_bw (Mb/s) udp_bw(Mb/s) Kube-OVN Overlay 15.2 14.6 23.6 2.65 Kube-OVN Underlay 14.3 13.8 24.2 3.46 Calico IPIP 21.4 20.2 23.6 1.18 Calico NoEncap 19.3 16.9 23.6 1.76 HOST Network 16.6 15.4 24.8 2.64 <p><code>qperf -t 60 &lt;server ip&gt; -ub -oo msg_size:1K -vu tcp_lat tcp_bw udp_lat udp_bw</code></p> Type tcp_lat (us) udp_lat (us) tcp_bw (Gb/s) udp_bw(Gb/s) Kube-OVN Overlay 16.5 15.8 10.2 2.77 Kube-OVN Underlay 15.9 14.5 9.6 3.22 Calico IPIP 22.5 21.5 1.45 1.14 Calico NoEncap 19.4 18.3 3.76 1.63 HOST Network 18.1 16.6 9.32 2.66 <p><code>qperf -t 60 &lt;server ip&gt; -ub -oo msg_size:4K -vu tcp_lat tcp_bw udp_lat udp_bw</code></p> Type tcp_lat (us) udp_lat (us) tcp_bw (Gb/s) udp_bw(Gb/s) Kube-OVN Overlay 34.7 41.6 16.0 9.23 Kube-OVN Underlay 32.6 44 15.1 6.71 Calico IPIP 44.8 52.9 2.94 3.26 Calico NoEncap 40 49.6 6.56 4.19 HOST Network 35.9 45.9 14.6 5.59 <p>In some cases the container network outperforms the host network, this is because the container network path is optimized to completely bypass netfilter. Due to the existence of <code>kube-proxy</code>, all packets in host network have to go through netfilter, which will lead to more CPU consumption, so that container network in some environments has better performance.</p>"},{"location":"en/advance/performance-tuning/#dataplane-performance-optimization-methods","title":"Dataplane performance optimization methods","text":"<p>The optimization methods described here are related to the hardware and software environment and the desired functionality, so please carefully understand the prerequisites for optimization before attempting it.</p>"},{"location":"en/advance/performance-tuning/#cpu-performance-mode-tuning","title":"CPU Performance Mode Tuning","text":"<p>In some environments the CPU is running in power saving mode, performance in this mode will be unstable and latency will increase significantly, it is recommended to use the CPU's performance mode for more stable performance.</p> <pre><code>cpupower frequency-set -g performance\n</code></pre>"},{"location":"en/advance/performance-tuning/#nic-hardware-queue-adjustment","title":"NIC Hardware Queue Adjustment","text":"<p>In the case of increased traffic, a small buffer queue may lead to significant performance degradation due to a high packet loss rate and needs to be tuned.</p> <p>Check the current NIC queue length:</p> <pre><code># ethtool -g eno1\nRing parameters for eno1:\n Pre-set maximums:\n RX:             4096\nRX Mini:        0\nRX Jumbo:       0\nTX:             4096\nCurrent hardware settings:\n RX:             255\nRX Mini:        0\nRX Jumbo:       0\nTX:             255\n</code></pre> <p>Increase the queue length to the maximum:</p> <pre><code>ethtool -G eno1 rx 4096\nethtool -G eno1 tx 4096\n</code></pre>"},{"location":"en/advance/performance-tuning/#optimize-with-tuned","title":"Optimize with tuned","text":"<p>tuned can use a series of preconfigured profile files to perform system optimizations for a specific scenario.</p> <p>For latency-first scenarios:</p> <pre><code>tuned-adm profile network-latency\n</code></pre> <p>For throughput-first scenarios:</p> <pre><code>tuned-adm profile network-throughput\n</code></pre>"},{"location":"en/advance/performance-tuning/#interrupt-binding","title":"Interrupt Binding","text":"<p>We recommend disabling <code>irqbalance</code> and binding NIC interrupts to specific CPUs to avoid performance fluctuations caused by switching between multiple CPUs.</p>"},{"location":"en/advance/performance-tuning/#disable-ovn-lb","title":"Disable OVN LB","text":"<p>The L2 LB implementation of OVN requires calling the kernel's <code>conntrack</code> module and recirculate, resulting in a significant CPU overhead, which is tested to be around 20%. For Overlay networks you can use <code>kube-proxy</code> to complete the service forwarding function for better Pod-to-Pod performance. This can be turned off in <code>kube-ovn-controller</code> args:</p> <pre><code>command:\n- /kube-ovn/start-controller.sh\nargs:\n...\n- --enable-lb=false\n...\n</code></pre> <p>In Underlay mode <code>kube-proxy</code> cannot use iptables or ipvs to control container network traffic, if you want to disable the LB function, you need to confirm whether you do not need the Service function.</p>"},{"location":"en/advance/performance-tuning/#fastpath-kernel-module","title":"FastPath Kernel Module","text":"<p>Since the container network and the host network are on different network ns, the packets will pass through the netfilter module several times when they are transmitted across the host, which results in a CPU overhead of nearly 20%. The <code>FastPath</code> module can reduce CPU overhead by bypassing netfilter, since in most cases applications within a container network do not need to use the functionality of the netfilter module.</p> <p>If you need to use the functions provided by netfilter such as iptables, ipvs, nftables, etc. in the container network, this module will disable the related functions.</p> <p>Since kernel modules are kernel version dependent, it is not possible to provide a single kernel module artifact that adapts to all kernels. We pre-compiled the <code>FastPath</code> module for part of the kernels, which can be accessed by tunning-package.</p> <p>You can also compile it manually, see Compiling FastPath Module</p> <p>After obtaining the kernel module, you can load the <code>FastPath</code> module on each node using <code>insmod kube_ovn_fastpath.ko</code> and verify that the module was loaded successfully using <code>dmesg</code>:</p> <pre><code># dmesg\n...\n[619631.323788] init_module,kube_ovn_fastpath_local_out\n[619631.323798] init_module,kube_ovn_fastpath_post_routing\n[619631.323800] init_module,kube_ovn_fastpath_pre_routing\n[619631.323801] init_module,kube_ovn_fastpath_local_in\n...\n</code></pre>"},{"location":"en/advance/performance-tuning/#ovs-kernel-module-optimization","title":"OVS Kernel Module Optimization","text":"<p>OVS flow processing including hashing, matching, etc. consumes about 10% of the CPU resources. Some instruction sets on modern x86 CPUs such as <code>popcnt</code> and <code>sse4.2</code> can speed up the computation process, but the kernel is not compiled with these options enabled. It has been tested that the CPU consumption of flow-related operations is reduced to about 5% when the corresponding instruction set optimizations are enabled.</p> <p>Similar to the compilation of the <code>FastPath</code> module, it is not possible to provide a single kernel module artifact for all kernels. Users need to compile manually or go to tunning-package to see if a compiled package is available for download.</p> <p>Before using this kernel module, please check if the CPU supports the following instruction set:</p> <pre><code>cat /proc/cpuinfo  | grep popcnt\ncat /proc/cpuinfo  | grep sse4_2\n</code></pre>"},{"location":"en/advance/performance-tuning/#compile-and-install-in-centos","title":"Compile and Install in CentOS","text":"<p>Install the relevant compilation dependencies and kernel headers:</p> <pre><code>yum install -y gcc kernel-devel-$(uname -r) python3 autoconf automake libtool rpm-build openssl-devel\n</code></pre> <p>Compile the OVS kernel module and generate the corresponding RPM:</p> <pre><code>git clone -b branch-2.17 --depth=1 https://github.com/openvswitch/ovs.git\ncd ovs\ncurl -s  https://github.com/kubeovn/ovs/commit/2d2c83c26d4217446918f39d5cd5838e9ac27b32.patch |  git apply\n./boot.sh\n./configure --with-linux=/lib/modules/$(uname -r)/build CFLAGS=\"-g -O2 -mpopcnt -msse4.2\"\nmake rpm-fedora-kmod\ncd rpm/rpmbuild/RPMS/x86_64/\n</code></pre> <p>Copy the RPM to each node and install:</p> <pre><code>rpm -i openvswitch-kmod-2.15.2-1.el7.x86_64.rpm\n</code></pre> <p>If you have previously started Kube-OVN and the older version of the OVS module has been loaded into the kernel. It is recommended to reboot the machine to reload the new version of the kernel module.</p>"},{"location":"en/advance/performance-tuning/#compile-and-install-in-ubuntu","title":"Compile and Install in Ubuntu","text":"<p>Install the relevant compilation dependencies and kernel headers:</p> <pre><code>apt install -y autoconf automake libtool gcc build-essential libssl-dev\n</code></pre> <p>Compile the OVS kernel module and install:</p> <pre><code>apt install -y autoconf automake libtool gcc build-essential libssl-dev\n\ngit clone -b branch-2.17 --depth=1 https://github.com/openvswitch/ovs.git\ncd ovs\ncurl -s  https://github.com/kubeovn/ovs/commit/2d2c83c26d4217446918f39d5cd5838e9ac27b32.patch |  git apply\n./boot.sh\n./configure --prefix=/usr/ --localstatedir=/var --enable-ssl --with-linux=/lib/modules/$(uname -r)/build\nmake -j `nproc`\nmake install\nmake modules_install\n\ncat &gt; /etc/depmod.d/openvswitch.conf &lt;&lt; EOF\noverride openvswitch * extra\noverride vport-* * extra\nEOF\n\ndepmod -a\ncp debian/openvswitch-switch.init /etc/init.d/openvswitch-switch\n/etc/init.d/openvswitch-switch force-reload-kmod\n</code></pre> <p>If you have previously started Kube-OVN and the older version of the OVS module has been loaded into the kernel. It is recommended to reboot the machine to reload the new version of the kernel module.</p>"},{"location":"en/advance/performance-tuning/#using-stt-type-tunnel","title":"Using STT Type Tunnel","text":"<p>Common tunnel encapsulation protocols such as Geneve and Vxlan use the UDP protocol to encapsulate packets and are well supported in the kernel. However, when TCP packets are encapsulated using UDP, the optimization and offload features of modern operating systems and network cards for the TCP protocol do not work well, resulting in a significant drop in TCP throughput. In some virtualization scenarios, due to CPU limitations, TCP packet throughput may even be a tenth of that of the host network.</p> <p>STT provides an innovative tunneling protocol that uses TCP formatted header for encapsulation. This encapsulation only emulates the TCP protocol header format without actually establishing a TCP connection, but can take full advantage of the TCP optimization capabilities of modern operating systems and network cards. In our tests TCP packet throughput can be improved several times, reaching performance levels close to those of the host network.</p> <p>The STT tunnel is not pre-installed in the kernel and needs to be installed by compiling the OVS kernel module, which can be found in the previous section.</p> <p>Enable STT tunnel:</p> <pre><code>kubectl set env daemonset/ovs-ovn -n kube-system TUNNEL_TYPE=stt\n\nkubectl delete pod -n kube-system -lapp=ovs\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/security-group/","title":"SecurityGroup Usage","text":"<p>Kube-OVN has supported the configuration of security-groups, and the CRD used to configure security-groups is SecurityGroup.</p>"},{"location":"en/advance/security-group/#securitygroup-example","title":"SecurityGroup Example","text":"<pre><code>apiVersion: kubeovn.io/v1\nkind: SecurityGroup\nmetadata:\nname: sg-example\nspec:\nallowSameGroupTraffic: true\negressRules:\n- ipVersion: ipv4\npolicy: allow\npriority: 1\nprotocol: all\nremoteAddress: 10.16.0.13/16\nremoteType: address\ningressRules:\n- ipVersion: ipv4\npolicy: deny\npriority: 1\nprotocol: icmp\nremoteAddress: 10.16.0.14/16\nremoteType: address\n</code></pre> <p>The specific meaning of each field of the SecurityGroup can be found in the Kube-OVN API Reference.</p> <p>Pods bind security-groups by adding annotations, two annotations are used.</p> <pre><code>    ovn.kubernetes.io/port_security: \"true\"\novn.kubernetes.io/security_groups: sg-example\n</code></pre>"},{"location":"en/advance/security-group/#caution","title":"Caution","text":"<ul> <li> <p>Security-groups are finally restricted by setting ACL rules, and as mentioned in the OVN documentation, if two ACL rules match with the same priority, it is uncertain which ACL will actually work. Therefore, when setting up security-group rules, you need to be careful to differentiate the priority.</p> </li> <li> <p>When adding a security-group, it is important to know what restrictions are being added. As a CNI, Kube-OVN will perform a Pod-to-Gateway connectivity test after creating a Pod.</p> </li> </ul>"},{"location":"en/advance/security-group/#actual-test","title":"Actual test","text":"<p>Create a Pod using the following yaml, and specify the security-group in the annotation for the pod.</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nlabels:\napp: static\nannotations:\novn.kubernetes.io/port_security: 'true'\novn.kubernetes.io/security_groups: 'sg-example'\nname: sg-test-pod\nnamespace: default\nspec:\nnodeName: kube-ovn-worker\ncontainers:\n- image: docker.io/library/nginx:alpine\nimagePullPolicy: IfNotPresent\nname: qatest\n</code></pre> <p>The actual test results show as follows:</p> <pre><code># kubectl get pod -o wide\nNAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES\nsg-test-pod 0/1 ContainerCreating 0 5h32m &lt;none&gt; kube-ovn-worker &lt;none&gt; &lt;none&gt;\ntest-99fff7f86-52h9r 1/1 Running 0 5h41m 10.16.0.14 kube-ovn-control-plane &lt;none&gt; &lt;none&gt;\ntest-99fff7f86-qcgjw 1/1 Running 0 5h43m 10.16.0.13 kube-ovn-worker &lt;none&gt; &lt;none&gt;\n</code></pre> <p>Execute <code>kubectl describe pod</code> to see information about the pod, and you can see the error message:</p> <pre><code># kubectl describe pod sg-test-pod\nName: sg-test-pod\nNamespace: default\nPriority: 0\nNode: kube-ovn-worker/172.18.0.2\nStart Time: Tue, 28 Feb 2023 10:29:36 +0800\nLabels: app=static\nAnnotations: ovn.kubernetes.io/allocated: true\novn.kubernetes.io/cidr: 10.16.0.0/16\n              ovn.kubernetes.io/gateway: 10.16.0.1\n              ovn.kubernetes.io/ip_address: 10.16.0.15\n              ovn.kubernetes.io/logical_router: ovn-cluster\n              ovn.kubernetes.io/logical_switch: ovn-default\n              ovn.kubernetes.io/mac_address: 00:00:00:FA:17:97\n              ovn.kubernetes.io/pod_nic_type: veth-pair\n              ovn.kubernetes.io/port_security: true\novn.kubernetes.io/routed: true\novn.kubernetes.io/security_groups: sg-allow-reject\nStatus: Pending\nIP:\nIPs: &lt;none&gt;\n-\n- -\n- -\nEvents:\n  Type Reason Age From Message\n  ---- ------ ---- ---- -------\n  Warning FailedCreatePodSandBox 5m3s (x70 over 4h59m) kubelet (combined from similar events): Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox \"40636e0c7f1ade5500fa958486163d74f2e2300051a71522a9afd7ba0538afb6\": plugin type=\"kube-ovn\" failed ( add): RPC failed; request ip return 500 configure nic failed 10.16.0.15 network not ready after 200 ping 10.16.0.1\n</code></pre> <p>Modify the rules for the security group to add access rules to the gateway, refer to the following:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: SecurityGroup\nmetadata:\nname: sg-gw-both\nspec:\nallowSameGroupTraffic: true\negressRules:\n- ipVersion: ipv4\npolicy: allow\npriority: 2\nprotocol: all\nremoteAddress: 10.16.0.13/16\nremoteType: address\n- ipVersion: ipv4\npolicy: allow\npriority: 1\nprotocol: all\nremoteAddress: 10.16.0.1/16\nremoteType: address\ningressRules:\n- ipVersion: ipv4\npolicy: deny\npriority: 2\nprotocol: icmp\nremoteAddress: 10.16.0.14/16\nremoteType: address\n- ipVersion: ipv4\npolicy: allow\npriority: 1\nprotocol: icmp\nremoteAddress: 10.16.0.1/16\nremoteType: address\n</code></pre> <p>In the inbound and outbound rules respectively, add a rule to allow access to the gateway, and set the rule to have the highest priority.</p> <p>Deploying with the following yaml to bind security group, confirm that the Pod is operational:</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nlabels:\napp: static\nannotations:\novn.kubernetes.io/port_security: 'true'\novn.kubernetes.io/security_groups: 'sg-gw-both'\nname: sg-gw-both\nnamespace: default\nspec:\nnodeName: kube-ovn-worker\ncontainers:\n- image: docker.io/library/nginx:alpine\nimagePullPolicy: IfNotPresent\nname: qatest\n</code></pre> <p>To view Pod information after deployment:</p> <pre><code># kubectl get pod -o wide\nNAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES\nsg-test-pod 0/1 ContainerCreating 0 5h41m &lt;none&gt; kube-ovn-worker &lt;none&gt; &lt;none&gt;\nsg-gw-both 1/1 Running 0 5h37m 10.16.0.19 kube-ovn-worker &lt;none&gt; &lt;none&gt;\n</code></pre> <p>So for the use of security groups, be particularly clear about the effect of the added restriction rules. If it is simply to restrict traffic access, consider using a network policy instead.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/vip/","title":"VIP Reservation","text":"<p>In some scenarios we want to dynamically reserve part of the IP but not assign it to Pods but to other infrastructure e.g:</p> <ul> <li>Kubernetes nested Kubernetes scenarios where the upper Kubernetes uses the Underlay network take up the available addresses of the underlying Subnet.</li> <li>LB or other network infrastructure requires the use of an IP within a Subnet.</li> </ul>"},{"location":"en/advance/vip/#create-random-address-vip","title":"Create Random Address VIP","text":"<p>If you just want to set aside a number of IPs and have no requirement for the IP addresses themselves, you can use the following yaml to create them:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Vip\nmetadata:\nname: vip-dynamic-01\nspec:\nsubnet: ovn-default\ntype: \"\"\n</code></pre> <ul> <li><code>subnet</code>: reserve the IP from this Subnet.</li> <li><code>type</code>: Currently, two types are supported. If the value is empty, it indicates that it is only used for occupying ip addresses of ipam. <code>switch_lb_vip</code> The front-end vip address and back-end ip address of the switch lb must be on the same subnet.</li> </ul> <p>Query the VIP after creation.</p> <pre><code># kubectl get vip\nNAME             V4IP         PV4IP   MAC                 PMAC   V6IP   PV6IP   SUBNET        READY\nvip-dynamic-01   10.16.0.12           00:00:00:F0:DB:25                         ovn-default   true\n</code></pre> <p>It can be seen that the VIP is assigned the IP address <code>10.16.0.12</code>, which can later be used by other network infrastructures.</p>"},{"location":"en/advance/vip/#create-a-fixed-address-vip","title":"Create a fixed address VIP","text":"<p>The IP address of the reserved VIP can be fixed using the following yaml:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Vip\nmetadata:\nname: static-vip01\nspec:\nsubnet: ovn-default V4ip: \"10.16.0.121\"\n</code></pre> <ul> <li><code>subnet</code>: reserve the IP from this Subnet.</li> <li><code>V4ip</code>: A fixed-assigned IP address that should within the CIDR range of <code>subnet</code>.</li> </ul> <p>Query the VIP after creation:</p> <pre><code># kubectl get vip\nNAME             V4IP         PV4IP   MAC                 PMAC   V6IP   PV6IP   SUBNET        READY\nstatic-vip01   10.16.0.121           00:00:00:F0:DB:26                         ovn-default   true\n</code></pre> <p>It can be seen that the VIP has been assigned the expected IP address.</p>"},{"location":"en/advance/vip/#statefulset-kubevirt-vm-keep-vip","title":"StatefulSet &amp; Kubevirt VM keep VIP","text":"<p>Specify for <code>StatefulSet</code> and <code>VM</code> resources, these Pods their owned will reuse the VIP when these Pods recreating.</p> <p>VM keep VIP must be enable the <code>keep-vm-ip</code> param in <code>kube-ovn-controller</code>. Refer Kubevirt VM Fixed Address Settings</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/vpc-internal-lb/","title":"Customize VPC Internal Load Balancing","text":"<p>The Service provided by Kubernetes can be used for load balancing within the cluster. However, there are several issues with using Service as internal load balancing in customize VPC mode:</p> <ol> <li>The Service IP range is a cluster resource, shared by all customize VPCs, and cannot overlap.</li> <li>Users cannot set internal load balancing IP addresses according to their own preferences.</li> </ol> <p>To address the above issues, Kube OVN introduced the <code>SwitchLBRule</code> CRD in 1.11, allowing users to set internal load balancing rules within customize VPCs.</p> <p><code>SwitchLBRule</code> support the following two ways to set internal load balancing rules within a customize VPC.</p>"},{"location":"en/advance/vpc-internal-lb/#automatically-generate-load-balancing-rules-by-selector","title":"Automatically Generate Load Balancing Rules by <code>Selector</code>","text":"<p>Load balancing rules can be generated by <code>selector</code> automatic association with <code>pod</code> configuration through <code>label</code>.</p> <p>example of <code>SwitchLBRule</code> is as follows:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: SwitchLBRule\nmetadata:\nname:  cjh-slr-nginx\nspec:\nvip: 1.1.1.1\nsessionAffinity: ClientIP\nnamespace: default\nselector:\n- app:nginx\nports:\n- name: dns\nport: 8888\ntargetPort: 80\nprotocol: TCP\n</code></pre> <ul> <li> <p>usage of <code>selector</code>, <code>sessionAffinity</code>, and <code>port</code> is the same as Kubernetes Service.</p> </li> <li> <p><code>vip</code>\uff1acustomize load balancing IP address.</p> </li> <li> <p><code>namespace</code>\uff1anamespace of the <code>pod</code> selected by <code>selector</code>.</p> <p>Kube OVN will determine the VPC of the selected <code>pod</code> based on the <code>SwitchLBRule</code> definition and set the corresponding L2 LB.</p> </li> </ul>"},{"location":"en/advance/vpc-internal-lb/#manually-defined-load-balancing-rules-by-endpoints","title":"Manually Defined Load Balancing Rules by <code>Endpoints</code>","text":"<p>Load balancing rules can be customized configured by   <code>endpoints</code>, to support scenarios where load balancing rules cannot be automatically generated through <code>selector</code>.  For example, the load balancing backend is <code>vm</code> created by <code>kubevirt</code>.</p> <p>example of <code>SwitchLBRule</code> is as follows:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: SwitchLBRule\nmetadata:\nname:  cjh-slr-nginx\nspec:\nvip: 1.1.1.1\nsessionAffinity: ClientIP\nnamespace: default\nendpoints:\n- 192.168.0.101\n- 192.168.0.102\n- 192.168.0.103\nports:\n- name: dns\nport: 8888\ntargetPort: 80\nprotocol: TCP\n</code></pre> <ul> <li> <p>usage of <code>sessionAffinity</code>, and <code>port</code> is the same as Kubernetes Service.</p> </li> <li> <p><code>vip</code>\uff1acustomize load balancing IP address.</p> </li> <li> <p><code>namespace</code>\uff1anamespace of the <code>pod</code> selected by <code>selector</code>.</p> </li> <li> <p><code>endpoints</code>\uff1aload balancing backend IP list.</p> </li> </ul> <p>attention\uff1aIf both <code>selector</code> and <code>endpoints</code> are configured, the <code>selector</code> configuration will be automatically ignored.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/vpc-peering/","title":"VPC Peering","text":"<p>VPC peering provides a mechanism for bridging two VPC networks through logical routes so that workloads within two VPCs can access each other through private addresses as if they were on the same private network, without the need for NAT forwarding through a gateway.</p>"},{"location":"en/advance/vpc-peering/#prerequisites","title":"Prerequisites","text":"<ol> <li>This feature is only available for customized VPCs.</li> <li>To avoid route overlap the subnet CIDRs within the two VPCs cannot overlap.</li> <li>Currently, only interconnection of two VPCs is supported.</li> </ol>"},{"location":"en/advance/vpc-peering/#usage","title":"Usage","text":"<p>First create two non-interconnected VPCs with one Subnet under each VPC, and the CIDRs of the Subnets do not overlap with each other.</p> <pre><code>kind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: vpc-1\nspec: {}\n---\nkind: Subnet\napiVersion: kubeovn.io/v1\nmetadata:\nname: net1\nspec:\nvpc: vpc-1\ncidrBlock: 10.0.0.0/16\n---\nkind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: vpc-2\nspec: {}\n---\nkind: Subnet\napiVersion: kubeovn.io/v1\nmetadata:\nname: net2\nspec:\nvpc: vpc-2\ncidrBlock: 172.31.0.0/16\n</code></pre> <p>Add <code>vpcPeerings</code> and the corresponding static routes within each VPC:</p> <pre><code>kind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: vpc-1\nspec: vpcPeerings:\n- remoteVpc: vpc-2\nlocalConnectIP: 169.254.0.1/30\nstaticRoutes:\n- cidr: 172.31.0.0/16\nnextHopIP: 169.254.0.2\npolicy: policyDst\n---\nkind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: vpc-2\nspec:\nvpcPeerings:\n- remoteVpc: vpc-1\nlocalConnectIP: 169.254.0.2/30\nstaticRoutes:\n- cidr: 10.0.0.0/16\nnextHopIP: 169.254.0.1\npolicy: policyDst\n</code></pre> <ul> <li><code>remoteVpc</code>: The name of another peering VPC.</li> <li><code>localConnectIP</code>: As the IP address and CIDR of the interconnection endpoint. Note that both IPs should belong to the same CIDR and should not conflict with existing subnets.</li> <li><code>cidr</code>\uff1aCIDR of the peering Subnet.</li> <li><code>nextHopIP</code>\uff1aThe <code>localConnectIP</code> on the other end of the peering VPC.</li> </ul> <p>Create Pods under the two Subnets</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nannotations:\novn.kubernetes.io/logical_switch: net1\nname: vpc-1-pod\nspec:\ncontainers:\n- name: vpc-1-pod\nimage: docker.io/library/nginx:alpine\n---\napiVersion: v1\nkind: Pod\nmetadata:\nannotations:\novn.kubernetes.io/logical_switch: net2\nname: vpc-2-pod\nspec:\ncontainers:\n- name: vpc-2-pod\nimage: docker.io/library/nginx:alpine\n</code></pre> <p>Test the network connectivity</p> <pre><code># kubectl exec -it vpc-1-pod -- ping $(kubectl get pod vpc-2-pod -o jsonpath='{.status.podIP}')\nPING 172.31.0.2 (172.31.0.2): 56 data bytes\n64 bytes from 172.31.0.2: seq=0 ttl=62 time=0.655 ms\n64 bytes from 172.31.0.2: seq=1 ttl=62 time=0.086 ms\n64 bytes from 172.31.0.2: seq=2 ttl=62 time=0.098 ms\n^C\n--- 172.31.0.2 ping statistics ---\n3 packets transmitted, 3 packets received, 0% packet loss\nround-trip min/avg/max = 0.086/0.279/0.655 ms\n# kubectl exec -it vpc-2-pod -- ping $(kubectl get pod vpc-1-pod -o jsonpath='{.status.podIP}')\nPING 10.0.0.2 (10.0.0.2): 56 data bytes\n64 bytes from 10.0.0.2: seq=0 ttl=62 time=0.594 ms\n64 bytes from 10.0.0.2: seq=1 ttl=62 time=0.093 ms\n64 bytes from 10.0.0.2: seq=2 ttl=62 time=0.088 ms\n^C\n--- 10.0.0.2 ping statistics ---\n3 packets transmitted, 3 packets received, 0% packet loss\nround-trip min/avg/max = 0.088/0.258/0.594 ms\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/windows/","title":"Windows Support","text":"<p>Kube-OVN supports Kubernetes cluster networks that include Windows system nodes, allowing unified containers network management.</p>"},{"location":"en/advance/windows/#prerequisites","title":"Prerequisites","text":"<ul> <li>Read Adding Windows nodes to add Windows nodes.</li> <li>Windows nodes must have the KB4489899 patch installed for Overlay/VXLAN networks to work properly, and it is recommended to update your system to the latest version.</li> <li>Hyper-V and management tools must be installed on the Windows node.</li> <li>Due to Windows restrictions tunnel encapsulation can only be used in Vxlan mode.</li> <li>SSL, IPv6, dual-stack, QoS features are not supported at this time.</li> <li>Dynamic subnet and dynamic tunnel interface are not supported at this time. You need to create the subnet and select the network interface before installing the Windows node.</li> <li>Multiple <code>ProviderNetwork</code>s are not supported, and the bridge interface configuration cannot be dynamically adjusted.</li> </ul>"},{"location":"en/advance/windows/#install-ovs-on-windows","title":"Install OVS on Windows","text":"<p>Due to some issues with upstream OVN and OVS support for Windows containers, a modified installation package provided by Kube-OVN is required.</p> <p>Use the following command to enable the <code>TESTSIGNING</code> startup item on the Windows node, which requires a system reboot to take effect.</p> <pre><code>bcdedit /set LOADOPTIONS DISABLE_INTEGRITY_CHECKS\nbcdedit /set TESTSIGNING ON\nbcdedit /set nointegritychecks ON\n</code></pre> <p>Download Windows package on Windows node and install.</p> <p>Confirm that the service is running properly after installation:</p> <pre><code>PS &gt; Get-Service | findstr ovs\nRunning  ovsdb-server  Open vSwitch DB Service\nRunning  ovs-vswitchd  Open vSwitch Service\n</code></pre>"},{"location":"en/advance/windows/#install-kube-ovn","title":"Install Kube-OVN","text":"<p>Download the installation script in the Windows node install.ps1.</p> <p>Add relevant parameters and run:</p> <pre><code>.\\install.ps1 -KubeConfig C:\\k\\admin.conf -ApiServer https://192.168.140.180:6443 -ServiceCIDR 10.96.0.0/12\n</code></pre> <p>By default, Kube-OVN uses the NIC where the node IP is located as the tunnel interface. If you need to use another NIC, you need to add the specified annotation to the Node before installation, e.g. <code>ovn.kubernetes.io/tunnel_interface=Ethernet1</code>.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/with-bgp/","title":"BGP Support","text":"<p>Kube-OVN supports broadcasting the IP address of the Pod or Subnet to the outside world via BGP protocol, so that the outside world can access the Pod directly through the Pod IP. To use this feature, you need to install <code>kube-ovn-speaker</code> on specific nodes and add the corresponding annotation to the Pod or Subnet that needs to be exposed to the outside world.</p>"},{"location":"en/advance/with-bgp/#install-kube-ovn-speaker","title":"Install kube-ovn-speaker","text":"<p><code>kube-ovn-speaker</code> use GoBGP to publish routing information to the outside world and set the next-hop route to itself.</p> <p>Since the node where <code>kube-ovn-speaker</code> is deployed needs to carry return traffic, specific labeled nodes need to be selected for deployment:</p> <pre><code>kubectl label nodes speaker-node-1 ovn.kubernetes.io/bgp=true\nkubectl label nodes speaker-node-2 ovn.kubernetes.io/bgp=true\n</code></pre> <p>When there are multiple instances of kube-ovn-speaker, each of them will publish routes to the outside world, the upstream router needs to support multi-path ECMP.</p> <p>Download the corresponding yaml:</p> <pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/yamls/speaker.yaml\n</code></pre> <p>Modify the corresponding configuration in yaml:</p> <pre><code>--neighbor-address=10.32.32.1\n--neighbor-as=65030\n--cluster-as=65000\n</code></pre> <ul> <li><code>neighbor-address</code>: The address of the BGP Peer, usually the router gateway address.</li> <li><code>neighbor-as</code>: The AS number of the BGP Peer.</li> <li><code>cluster-as</code>: The AS number of the container network.</li> </ul> <p>Deploy yaml:</p> <pre><code>kubectl apply -f speaker.yaml\n</code></pre>"},{"location":"en/advance/with-bgp/#publish-podsubnet-routes","title":"Publish Pod/Subnet Routes","text":"<p>To use BGP for external routing, first set <code>natOutgoing</code> to <code>false</code> for the corresponding Subnet to allow the Pod IP to enter the underlying network directly.</p> <p>Add annotation to publish routes:</p> <pre><code>kubectl annotate pod sample ovn.kubernetes.io/bgp=true\nkubectl annotate subnet ovn-default ovn.kubernetes.io/bgp=true\n</code></pre> <p>Delete annotation to disable the publishing:</p> <pre><code>kubectl annotate pod perf-ovn-xzvd4 ovn.kubernetes.io/bgp-\nkubectl annotate subnet ovn-default ovn.kubernetes.io/bgp-\n</code></pre>"},{"location":"en/advance/with-bgp/#bgp-advance-options","title":"BGP Advance Options","text":"<p><code>kube-ovn-speaker</code> supports more BGP parameters for advanced configuration, which can be adjusted by users according to their network environment:</p> <ul> <li><code>announce-cluster-ip</code>: Whether to publish Service routes to the public, default is <code>false</code>.</li> <li><code>auth-password</code>: The access password for the BGP peer.</li> <li><code>holdtime</code>: The heartbeat detection time between BGP neighbors. Neighbors with no messages after the change time will be removed, the default is 90 seconds.</li> <li><code>graceful-restart</code>: Whether to enable BGP Graceful Restart.</li> <li><code>graceful-restart-time</code>: BGP Graceful restart time refer to RFC4724 3.</li> <li><code>graceful-restart-deferral-time</code>: BGP Graceful restart deferral time refer to RFC4724 4.1.</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/with-cilium/","title":"Integration with Cilium","text":"<p>Cilium is an eBPF-based networking and security component. Kube-OVN uses the CNI Chaining mode to enhance existing features. Users can use both the rich network abstraction capabilities of Kube-OVN and the monitoring and security capabilities that come with eBPF.</p> <p>By integrating Cilium, Kube-OVN users can have the following gains:</p> <ul> <li>Richer and more efficient security policies.</li> <li>Hubble-based monitoring and UI.</li> </ul> <p></p>"},{"location":"en/advance/with-cilium/#prerequisites","title":"Prerequisites","text":"<ol> <li>Linux kernel version above 4.19 or other compatible kernel for full eBPF capability support.</li> <li>Install Helm in advance to prepare for the installation of Cilium, please refer to Installing Helm to deploy Helm.</li> </ol>"},{"location":"en/advance/with-cilium/#configure-kube-ovn","title":"Configure Kube-OVN","text":"<p>In order to fully utilize the security capabilities of Cilium, you need to disable the <code>networkpolicy</code> feature within Kube-OVN and adjust the CNI configuration priority.</p> <p>Change the following variables in the <code>install.sh</code> script:</p> <pre><code>ENABLE_NP=false\nCNI_CONFIG_PRIORITY=10\n</code></pre> <p>If the deployment is complete, you can adjust the args of <code>kube-ovn-controller</code>:</p> <pre><code>args:\n- --enable-np=false\n</code></pre> <p>Modify the <code>kube-ovn-cni</code> args to adjust the CNI configuration priority:</p> <pre><code>args:\n- --cni-conf-name=10-kube-ovn.conflist\n</code></pre> <p>Adjust the Kube-OVN cni configuration name on each node:</p> <pre><code>mv /etc/cni/net.d/01-kube-ovn.conflist /etc/cni/net.d/10-kube-ovn.conflist\n</code></pre>"},{"location":"en/advance/with-cilium/#deploy-cilium","title":"Deploy Cilium","text":"<p>Create the <code>chaining.yaml</code> configuration file to use Cilium's <code>generic-veth</code> mode:</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: cni-configuration\nnamespace: kube-system\ndata:\ncni-config: |-\n{\n\"name\": \"generic-veth\",\n\"cniVersion\": \"0.3.1\",\n\"plugins\": [\n{\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\",\n\"ipam\": {\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\"\n}\n},\n{\n\"type\": \"portmap\",\n\"snat\": true,\n\"capabilities\": {\"portMappings\": true}\n},\n{\n\"type\": \"cilium-cni\"\n}\n]\n}\n</code></pre> <p>Installation the chaining config:</p> <pre><code>kubectl apply -f chaining.yaml\n</code></pre> <p>Deploying Cilium with Helm:</p> <pre><code>helm repo add cilium https://helm.cilium.io/\nhelm install cilium cilium/cilium --version 1.11.6 \\\n--namespace kube-system \\\n--set cni.chainingMode=generic-veth \\\n--set cni.customConf=true \\\n--set cni.configMap=cni-configuration \\\n--set tunnel=disabled \\\n--set enableIPv4Masquerade=false \\\n--set enableIdentityMark=false </code></pre> <p>Confirm that the Cilium installation was successful:</p> <pre><code># cilium  status\n/\u00af\u00af\\\n/\u00af\u00af\\__/\u00af\u00af\\    Cilium:         OK\n \\__/\u00af\u00af\\__/    Operator:       OK\n /\u00af\u00af\\__/\u00af\u00af\\    Hubble:         disabled\n \\__/\u00af\u00af\\__/    ClusterMesh:    disabled\n    \\__/\n\nDaemonSet         cilium             Desired: 2, Ready: 2/2, Available: 2/2\nDeployment        cilium-operator    Desired: 2, Ready: 2/2, Available: 2/2\nContainers:       cilium             Running: 2\ncilium-operator    Running: 2\nCluster Pods:     8/11 managed by Cilium\nImage versions    cilium             quay.io/cilium/cilium:v1.10.5@sha256:0612218e28288db360c63677c09fafa2d17edda4f13867bcabf87056046b33bb: 2\ncilium-operator    quay.io/cilium/operator-generic:v1.10.5@sha256:2d2f730f219d489ff0702923bf24c0002cd93eb4b47ba344375566202f56d972: 2\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/with-openstack/","title":"Integration with OpenStack","text":"<p>In some cases, users need to run virtual machines with OpenStack and containers with Kubernetes, and need the network to interoperate between containers and virtual machines and be under a unified control plane. If the OpenStack Neutron side also uses OVN as the underlying network, then Kube-OVN can use either cluster interconnection or shared underlying OVN to connect the OpenStack and Kubernetes networks.</p>"},{"location":"en/advance/with-openstack/#cluster-interconnection","title":"Cluster Interconnection","text":"<p>This pattern is similar to Cluster Inter-Connection with OVN-IC to connect two Kubernetes cluster networks, except that the two ends of the cluster are replaced with OpenStack and Kubernetes\u3002</p>"},{"location":"en/advance/with-openstack/#prerequisites","title":"Prerequisites","text":"<ol> <li>The subnet CIDRs within OpenStack and Kubernetes cannot overlap with each other in auto-route mode.</li> <li>A set of machines needs to exist that can be accessed by each cluster over the network and used to deploy controllers that interconnect across clusters.</li> <li>Each cluster needs to have a set of machines that can access each other across clusters via IP as the gateway nodes.</li> <li>This solution only connects to the Kubernetes default subnet with selected VPC in OpenStack.</li> </ol>"},{"location":"en/advance/with-openstack/#deploy-ovn-ic-db","title":"Deploy OVN-IC DB","text":"<p>Start the <code>OVN-IC</code> DB with the following command:</p> <pre><code>docker run --name=ovn-ic-db -d --network=host -v /etc/ovn/:/etc/ovn -v /var/run/ovn:/var/run/ovn -v /var/log/ovn:/var/log/ovn kubeovn/kube-ovn:v1.13.0 bash start-ic-db.sh\n</code></pre>"},{"location":"en/advance/with-openstack/#kubernetes-side-operations","title":"Kubernetes Side Operations","text":"<p>Create <code>ovn-ic-config</code> ConfigMap in <code>kube-system</code> Namespace \uff1a</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: ovn-ic-config\nnamespace: kube-system\ndata:\nenable-ic: \"true\"\naz-name: \"az1\" ic-db-host: \"192.168.65.3\"\nic-nb-port: \"6645\" ic-sb-port: \"6646\"\ngw-nodes: \"az1-gw\"\nauto-route: \"true\"\n</code></pre> <ul> <li><code>enable-ic</code>: Whether to enable cluster interconnection.</li> <li><code>az-name</code>: Distinguish the cluster names of different clusters, each interconnected cluster needs to be different.</li> <li><code>ic-db-host</code>: Address of the node where the <code>OVN-IC</code> DB is deployed.</li> <li><code>ic-nb-port</code>: <code>OVN-IC</code> Northbound Database port, default 6645.</li> <li><code>ic-sb-port</code>: <code>OVN-IC</code> Southbound Database port, default 6645.</li> <li><code>gw-nodes</code>: The name of the nodes in the cluster interconnection that takes on the work of the gateways, separated by commas.</li> <li><code>auto-route</code>: Whether to automatically publish and learn routes.</li> </ul>"},{"location":"en/advance/with-openstack/#openstack-side-operations","title":"OpenStack Side Operations","text":"<p>Create logical routers that interconnect with Kubernetes:</p> <pre><code># openstack router create router0\n# openstack router list\n+--------------------------------------+---------+--------+-------+----------------------------------+\n| ID                                   | Name    | Status | State | Project                          |\n+--------------------------------------+---------+--------+-------+----------------------------------+\n| d5b38655-249a-4192-8046-71aa4d2b4af1 | router0 | ACTIVE | UP    | 98a29ab7388347e7b5ff8bdd181ba4f9 |\n+--------------------------------------+---------+--------+-------+----------------------------------+\n</code></pre> <p>Set the availability zone name in the OVN northbound database within OpenStack, which needs to be different from the other interconnected clusters:</p> <pre><code>ovn-nbctl set NB_Global . name=op-az\n</code></pre> <p>Start the <code>OVN-IC</code> controller at a node that has access to the <code>OVN-IC</code> DB:</p> <pre><code>/usr/share/ovn/scripts/ovn-ctl --ovn-ic-nb-db=tcp:192.168.65.3:6645 \\\n--ovn-ic-sb-db=tcp:192.168.65.3:6646 \\\n--ovn-northd-nb-db=unix:/run/ovn/ovnnb_db.sock \\\n--ovn-northd-sb-db=unix:/run/ovn/ovnsb_db.sock \\\nstart_ic\n</code></pre> <ul> <li><code>ovn-ic-nb-db</code>\uff0c<code>ovn-ic-sb-db</code>: OVN-IC Northbound database and southbound database addresses.</li> <li><code>ovn-northd-nb-db</code>\uff0c <code>ovn-northd-sb-db</code>: Current cluster OVN northbound database and southbound data address.</li> </ul> <p>Configuration gateway nodes:</p> <pre><code>ovs-vsctl set open_vswitch . external_ids:ovn-is-interconn=true\n</code></pre> <p>The next step is to create a logical topology by operating the OVN in OpenStack.</p> <p>Connect the <code>ts</code> interconnect switch and the <code>router0</code> logical router, and set the relevant rules:</p> <pre><code>ovn-nbctl lrp-add router0 lrp-router0-ts 00:02:ef:11:39:4f 169.254.100.73/24\novn-nbctl lsp-add ts lsp-ts-router0 -- lsp-set-addresses lsp-ts-router0 router \\\n-- lsp-set-type lsp-ts-router0 router \\\n-- lsp-set-options lsp-ts-router0  router-port=lrp-router0-ts\novn-nbctl lrp-set-gateway-chassis lrp-router0-ts {gateway chassis} 1000\novn-nbctl set NB_Global . options:ic-route-adv=true options:ic-route-learn=true\n</code></pre> <p>Verify that OpenStack has learned the Kubernetes routing rules:</p> <pre><code># ovn-nbctl lr-route-list router0\nIPv4 Routes\n                10.0.0.22            169.254.100.34 dst-ip (learned)\n10.16.0.0/16            169.254.100.34 dst-ip (learned)\n</code></pre> <p>Next, you can create a virtual machine under the <code>router0</code> network to verify that it can interconnect with Pods under Kubernetes.</p>"},{"location":"en/advance/with-openstack/#shared-underlay-ovn","title":"Shared Underlay OVN","text":"<p>In this scenario, OpenStack and Kubernetes share the same OVN, so concepts such as VPC and Subnet can be pulled together for better control and interconnection.</p> <p>In this mode we deploy the OVN normally using Kube-OVN, and OpenStack modifies the Neutron configuration to connect to the same OVN DB. OpenStack requires networking-ovn as a Neutron backend implementation.</p>"},{"location":"en/advance/with-openstack/#neutron-modification","title":"Neutron Modification","text":"<p>Modify the Neutron configuration file <code>/etc/neutron/plugins/ml2/ml2_conf.ini</code>\uff1a</p> <pre><code>[ovn]\n...\novn_nb_connection = tcp:[192.168.137.176]:6641,tcp:[192.168.137.177]:6641,tcp:[192.168.137.178]:6641\novn_sb_connection = tcp:[192.168.137.176]:6642,tcp:[192.168.137.177]:6642,tcp:[192.168.137.178]:6642\novn_l3_scheduler = OVN_L3_SCHEDULER\n</code></pre> <ul> <li><code>ovn_nb_connection</code>\uff0c <code>ovn_sb_connection</code>: The address needs to be changed to the address of the <code>ovn-central</code> nodes deployed by Kube-OVN.</li> </ul> <p>Modify the OVS configuration for each node:</p> <pre><code>ovs-vsctl set open . external-ids:ovn-remote=tcp:[192.168.137.176]:6642,tcp:[192.168.137.177]:6642,tcp:[192.168.137.178]:6642\novs-vsctl set open . external-ids:ovn-encap-type=geneve\novs-vsctl set open . external-ids:ovn-encap-ip=192.168.137.200\n</code></pre> <ul> <li><code>external-ids:ovn-remote</code>: The address needs to be changed to the address of the <code>ovn-central</code> nodes deployed by Kube-OVN.</li> <li><code>ovn-encap-ip</code>: Change to the IP address of the current node.</li> </ul>"},{"location":"en/advance/with-openstack/#using-openstack-internal-resources-in-kubernetes","title":"Using OpenStack Internal Resources in Kubernetes","text":"<p>The next section describes how to query OpenStack's network resources in Kubernetes and create Pods in the subnet from OpenStack.</p> <p>Query the existing network resources in OpenStack for the following resources that have been pre-created.</p> <pre><code># openstack router list\n+--------------------------------------+---------+--------+-------+----------------------------------+\n| ID                                   | Name    | Status | State | Project                          |\n+--------------------------------------+---------+--------+-------+----------------------------------+\n| 22040ed5-0598-4f77-bffd-e7fd4db47e93 | router0 | ACTIVE | UP    | 62381a21d569404aa236a5dd8712449c |\n+--------------------------------------+---------+--------+-------+----------------------------------+\n# openstack network list\n+--------------------------------------+----------+--------------------------------------+\n| ID                                   | Name     | Subnets                              |\n+--------------------------------------+----------+--------------------------------------+\n| cd59e36a-37db-4c27-b709-d35379a7920f | provider | 01d73d9f-fdaa-426c-9b60-aa34abbfacae |\n+--------------------------------------+----------+--------------------------------------+\n# openstack subnet list\n+--------------------------------------+-------------+--------------------------------------+----------------+\n| ID                                   | Name        | Network                              | Subnet         |\n+--------------------------------------+-------------+--------------------------------------+----------------+\n| 01d73d9f-fdaa-426c-9b60-aa34abbfacae | provider-v4 | cd59e36a-37db-4c27-b709-d35379a7920f | 192.168.1.0/24 |\n+--------------------------------------+-------------+--------------------------------------+----------------+\n# openstack server list\n+--------------------------------------+-------------------+--------+-----------------------+--------+--------+\n| ID                                   | Name              | Status | Networks              | Image  | Flavor |\n+--------------------------------------+-------------------+--------+-----------------------+--------+--------+\n| 8433d622-a8d6-41a7-8b31-49abfd64f639 | provider-instance | ACTIVE | provider=192.168.1.61 | ubuntu | m1     |\n+--------------------------------------+-------------------+--------+-----------------------+--------+--------+\n</code></pre> <p>On the Kubernetes side, query the VPC resources from OpenStack:</p> <pre><code># kubectl get vpc\nNAME                                           STANDBY   SUBNETS\nneutron-22040ed5-0598-4f77-bffd-e7fd4db47e93   true      [\"neutron-cd59e36a-37db-4c27-b709-d35379a7920f\"]\novn-cluster                                    true      [\"join\",\"ovn-default\"]\n</code></pre> <p><code>neutron-22040ed5-0598-4f77-bffd-e7fd4db47e93</code> is the VPC resources synchronized from OpenStack.</p> <p>Next, you can create Pods and run them according to Kube-OVN's native VPC and Subnet operations.</p> <p>Bind VPC, Subnet to Namespace <code>net2</code> and create Pod:</p> <pre><code>apiVersion: v1\nkind: Namespace\nmetadata:\nname: net2\n---\napiVersion: kubeovn.io/v1\nkind: Vpc\nmetadata:\ncreationTimestamp: \"2021-06-20T13:34:11Z\"\ngeneration: 2\nlabels:\novn.kubernetes.io/vpc_external: \"true\"\nname: neutron-22040ed5-0598-4f77-bffd-e7fd4db47e93\nresourceVersion: \"583728\"\nuid: 18d4c654-f511-4def-a3a0-a6434d237c1e\nspec:\nnamespaces:\n- net2\n---\nkind: Subnet\napiVersion: kubeovn.io/v1\nmetadata:\nname: net2\nspec:\nvpc: neutron-22040ed5-0598-4f77-bffd-e7fd4db47e93\nnamespaces:\n- net2\ncidrBlock: 12.0.1.0/24\nnatOutgoing: false\n---\napiVersion: v1\nkind: Pod\nmetadata:\nname: ubuntu\nnamespace: net2\nspec:\ncontainers:\n- image: docker.io/kubeovn/kube-ovn:v1.8.0\ncommand:\n- \"sleep\"\n- \"604800\"\nimagePullPolicy: IfNotPresent\nname: ubuntu\nrestartPolicy: Always\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/with-ovn-ic/","title":"Cluster Inter-Connection with OVN-IC","text":"<p>Kube-OVN supports interconnecting two Kubernetes cluster Pod networks via OVN-IC, and the Pods in the two clusters can communicate directly via Pod IPs . Kube-OVN uses tunnels to encapsulate cross-cluster traffic, allowing container networks to interconnect between two clusters as long as there is a set of IP reachable machines.</p> <p>This mode of multi-cluster interconnection is for Overlay network. For Underlay network, it needs the underlying infrastructure to do the inter-connection work.</p> <p></p>"},{"location":"en/advance/with-ovn-ic/#prerequisites","title":"Prerequisites","text":"<ol> <li>The subnet CIDRs within OpenStack and Kubernetes cannot overlap with each other in auto-interconnect mode.    If there is overlap, you need to refer to the subsequent manual interconnection process, which can only connect non-overlapping Subnets.</li> <li>A set of machines needs to exist that can be accessed by each cluster over the network and used to deploy controllers that interconnect across clusters.</li> <li>Each cluster needs to have a set of machines that can access each other across clusters via IP as the gateway nodes.</li> <li>This solution only connects to the Kubernetes default VPCs.</li> </ol>"},{"location":"en/advance/with-ovn-ic/#deploy-a-single-node-ovn-ic-db","title":"Deploy a single-node OVN-IC DB","text":"<p>Deploy the <code>OVN-IC</code> DB on a machine accessible by <code>kube-ovn-controller</code>, This DB will hold the network configuration information synchronized up from each cluster.</p> <p>An environment deploying <code>docker</code> can start the <code>OVN-IC</code> DB with the following command.</p> <pre><code>docker run --name=ovn-ic-db -d --network=host --privileged -v /etc/ovn/:/etc/ovn -v /var/run/ovn:/var/run/ovn -v /var/log/ovn:/var/log/ovn kubeovn/kube-ovn:v1.13.0 bash start-ic-db.sh\n</code></pre> <p>For deploying a <code>containerd</code> environment instead of <code>docker</code> you can use the following command:</p> <pre><code>ctr -n k8s.io run -d --net-host --privileged --mount=\"type=bind,src=/etc/ovn/,dst=/etc/ovn,options=rbind:rw\" --mount=\"type=bind,src=/var/run/ovn,dst=/var/run/ovn,options=rbind:rw\" --mount=\"type=bind,src=/var/log/ovn,dst=/var/log/ovn,options=rbind:rw\" docker.io/kubeovn/kube-ovn:v1.13.0 ovn-ic-db bash start-ic-db.sh\n</code></pre>"},{"location":"en/advance/with-ovn-ic/#automatic-routing-mode","title":"Automatic Routing Mode","text":"<p>In auto-routing mode, each cluster synchronizes the CIDR information of the Subnet under its own default VPC to <code>OVN-IC</code>, so make sure there is no overlap between the Subnet CIDRs of the two clusters.</p> <p>Create <code>ovn-ic-config</code> ConfigMap in <code>kube-system</code> Namespace:</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: ovn-ic-config\nnamespace: kube-system\ndata:\nenable-ic: \"true\"\naz-name: \"az1\" ic-db-host: \"192.168.65.3\"\nic-nb-port: \"6645\" ic-sb-port: \"6646\"\ngw-nodes: \"az1-gw\"\nauto-route: \"true\"\n</code></pre> <ul> <li><code>enable-ic</code>: Whether to enable cluster interconnection.</li> <li><code>az-name</code>: Distinguish the cluster names of different clusters, each interconnected cluster needs to be different.</li> <li><code>ic-db-host</code>: Address of the node where the <code>OVN-IC</code> DB is deployed.</li> <li><code>ic-nb-port</code>: <code>OVN-IC</code> Northbound Database port, default 6645.</li> <li><code>ic-sb-port</code>: <code>OVN-IC</code> Southbound Database port, default 6645.</li> <li><code>gw-nodes</code>: The name of the nodes in the cluster interconnection that takes on the work of the gateways, separated by commas.</li> <li><code>auto-route</code>: Whether to automatically publish and learn routes.</li> </ul> <p>Note: To ensure the correct operation, the ConfigMap <code>ovn-ic-config</code> is not allowed to be modified. If any parameter needs to be changed, please delete this ConfigMap, modify it and then apply it again.</p> <p>Check if the interconnected logical switch <code>ts</code> has been established in the <code>ovn-ic</code> container with the following command\uff1a</p> <pre><code># ovn-ic-sbctl show\navailability-zone az1\n    gateway deee03e0-af16-4f45-91e9-b50c3960f809\n        hostname: az1-gw\n        type: geneve\n            ip: 192.168.42.145\n        port ts-az1\n            transit switch: ts\n            address: [\"00:00:00:50:AC:8C 169.254.100.45/24\"]\navailability-zone az2\n    gateway e94cc831-8143-40e3-a478-90352773327b\n        hostname: az2-gw\n        type: geneve\n            ip: 192.168.42.149\n        port ts-az2\n            transit switch: ts\n            address: [\"00:00:00:07:4A:59 169.254.100.63/24\"]\n</code></pre> <p>At each cluster observe if logical routes have learned peer routes:</p> <pre><code># kubectl ko nbctl lr-route-list ovn-cluster\nIPv4 Routes\n                10.42.1.1            169.254.100.45 dst-ip (learned)\n10.42.1.3                100.64.0.2 dst-ip\n                10.16.0.2                100.64.0.2 src-ip\n                10.16.0.3                100.64.0.2 src-ip\n                10.16.0.4                100.64.0.2 src-ip\n                10.16.0.6                100.64.0.2 src-ip\n             10.17.0.0/16            169.254.100.45 dst-ip (learned)\n100.65.0.0/16            169.254.100.45 dst-ip (learned)\n</code></pre> <p>Next, you can try <code>ping</code> a Pod IP in Cluster 1 directly from a Pod in Cluster 2 to see if you can work.</p> <p>For a subnet that does not want to automatically publish routes to the other end, you can disable route broadcasting by modifying <code>disableInterConnection</code> in the Subnet spec.</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: no-advertise\nspec:\ncidrBlock: 10.199.0.0/16\ndisableInterConnection: true\n</code></pre>"},{"location":"en/advance/with-ovn-ic/#manual-routing-mode","title":"Manual Routing Mode","text":"<p>For cases where there are overlapping CIDRs between clusters, and you only want to do partial subnet interconnection, you can manually publish subnet routing by following the steps below.</p> <p>Create <code>ovn-ic-config</code> ConfigMap in <code>kube-system</code> Namespace, and set <code>auto-route</code> to <code>false</code>:</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: ovn-ic-config\nnamespace: kube-system\ndata:\nenable-ic: \"true\"\naz-name: \"az1\" ic-db-host: \"192.168.65.3\"\nic-nb-port: \"6645\" ic-sb-port: \"6646\"\ngw-nodes: \"az1-gw\"\nauto-route: \"false\"\n</code></pre> <p>Find the address of the remote logical ports in each cluster separately, for later manual configuration of the route:</p> <pre><code>[root@az1 ~]# kubectl ko nbctl show\nswitch a391d3a1-14a0-4841-9836-4bd930c447fb (ts)\nport ts-az1\n        type: router\n        router-port: az1-ts\n    port ts-az2\n        type: remote\n        addresses: [\"00:00:00:4B:E2:9F 169.254.100.31/24\"]\n\n[root@az2 ~]# kubectl ko nbctl show\nswitch da6138b8-de81-4908-abf9-b2224ec4edf3 (ts)\nport ts-az2\n        type: router\n        router-port: az2-ts\n    port ts-az1\n        type: remote\n        addresses: [\"00:00:00:FB:2A:F7 169.254.100.79/24\"]        </code></pre> <p>The output above shows that the remote address from cluster <code>az1</code> to cluster <code>az2</code> is <code>169.254.100.31</code> and the remote address from <code>az2</code> to <code>az1</code> is <code>169.254.100.79</code>.</p> <p>In this example, the subnet CIDR within cluster <code>az1</code> is <code>10.16.0.0/24</code> and the subnet CIDR within cluster <code>az2</code> is <code>10.17.0.0/24</code>.</p> <p>Set up a route from cluster <code>az1</code> to cluster <code>az2</code> in cluster <code>az1</code>:</p> <pre><code>kubectl ko nbctl lr-route-add ovn-cluster 10.17.0.0/24 169.254.100.31\n</code></pre> <p>Set up a route to cluster <code>az1</code> in cluster <code>az2</code>:</p> <pre><code>kubectl ko nbctl lr-route-add ovn-cluster 10.16.0.0/24 169.254.100.79\n</code></pre>"},{"location":"en/advance/with-ovn-ic/#highly-available-ovn-ic-db-installation","title":"Highly Available OVN-IC DB Installation","text":"<p>A highly available cluster can be formed between <code>OVN-IC</code> DB via the Raft protocol, which requires a minimum of 3 nodes for this deployment model.</p> <p>First start the leader of the <code>OVN-IC</code> DB on the first node.</p> <p>Users deploying a <code>docker</code> environment can use the following command:</p> <pre><code>docker run --name=ovn-ic-db -d --network=host --privileged -v /etc/ovn/:/etc/ovn -v /var/run/ovn:/var/run/ovn -v /var/log/ovn:/var/log/ovn -e LOCAL_IP=\"192.168.65.3\"  -e NODE_IPS=\"192.168.65.3,192.168.65.2,192.168.65.1\"   kubeovn/kube-ovn:v1.13.0 bash start-ic-db.sh\n</code></pre> <p>If you are  using <code>containerd</code> you can use the following command:</p> <pre><code>ctr -n k8s.io run -d --net-host --privileged --mount=\"type=bind,src=/etc/ovn/,dst=/etc/ovn,options=rbind:rw\" --mount=\"type=bind,src=/var/run/ovn,dst=/var/run/ovn,options=rbind:rw\" --mount=\"type=bind,src=/var/log/ovn,dst=/var/log/ovn,options=rbind:rw\"  --env=\"NODE_IPS=\"192.168.65.3,192.168.65.2,192.168.65.1\"\" --env=\"LOCAL_IP=\"192.168.65.3\"\" docker.io/kubeovn/kube-ovn:v1.13.0 ovn-ic-db bash start-ic-db.sh\n</code></pre> <ul> <li><code>LOCAL_IP</code>\uff1a The IP address of the node where the current container is located.</li> <li><code>NODE_IPS</code>\uff1a The IP addresses of the three nodes running the <code>OVN-IC</code> database, separated by commas.</li> </ul> <p>Next, deploy the follower of the <code>OVN-IC</code> DB on the other two nodes.</p> <p><code>docker</code> environment can use the following command.</p> <pre><code>docker run --name=ovn-ic-db -d --network=host --privileged -v /etc/ovn/:/etc/ovn -v /var/run/ovn:/var/run/ovn -v /var/log/ovn:/var/log/ovn -e LOCAL_IP=\"192.168.65.2\"  -e NODE_IPS=\"192.168.65.3,192.168.65.2,192.168.65.1\" -e LEADER_IP=\"192.168.65.3\"  kubeovn/kube-ovn:v1.13.0 bash start-ic-db.sh\n</code></pre> <p>If using <code>containerd</code> you can use the following command:</p> <pre><code>ctr -n k8s.io run -d --net-host --privileged --mount=\"type=bind,src=/etc/ovn/,dst=/etc/ovn,options=rbind:rw\" --mount=\"type=bind,src=/var/run/ovn,dst=/var/run/ovn,options=rbind:rw\" --mount=\"type=bind,src=/var/log/ovn,dst=/var/log/ovn,options=rbind:rw\"  --env=\"NODE_IPS=\"192.168.65.3,192.168.65.2,192.168.65.1\"\" --env=\"LOCAL_IP=\"192.168.65.2\"\" --env=\"LEADER_IP=\"192.168.65.3\"\" docker.io/kubeovn/kube-ovn:v1.13.0 ovn-ic-db bash start-ic-db.sh\n</code></pre> <ul> <li><code>LOCAL_IP</code>\uff1a The IP address of the node where the current container is located.</li> <li><code>NODE_IPS</code>\uff1a The IP addresses of the three nodes running the <code>OVN-IC</code> database, separated by commas.</li> <li><code>LEADER_IP</code>: The IP address of the <code>OVN-IC</code> DB leader node.</li> </ul> <p>Specify multiple <code>OVN-IC</code> database node addresses when creating <code>ovn-ic-config</code> for each cluster:</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: ovn-ic-config\nnamespace: kube-system\ndata:\nenable-ic: \"true\"\naz-name: \"az1\" ic-db-host: \"192.168.65.3,192.168.65.2,192.168.65.1\"\nic-nb-port: \"6645\"\nic-sb-port: \"6646\"\ngw-nodes: \"az1-gw\"\nauto-route: \"true\"\n</code></pre>"},{"location":"en/advance/with-ovn-ic/#manual-reset","title":"Manual Reset","text":"<p>In some cases, the entire interconnection configuration needs to be cleaned up due to configuration errors, you can refer to the following steps to clean up your environment.</p> <p>Delete the current <code>ovn-ic-config</code> Configmap:</p> <pre><code>kubectl -n kube-system delete cm ovn-ic-config\n</code></pre> <p>Delete <code>ts</code> logical switch:</p> <pre><code>kubectl-ko nbctl ls-del ts\n</code></pre> <p>Repeat the same steps at the peer cluster.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/advance/with-submariner/","title":"Cluster Inter-Connection with Submariner","text":"<p>Submariner is an open source networking component that connects multiple Kubernetes cluster Pod and Service networks which can help Kube-OVN interconnect multiple clusters.</p> <p>Compared to OVN-IC, Submariner can connect Kube-OVN and non-Kube-OVN cluster networks, and Submariner can provide cross-cluster capability for services. However, Submariner currently only enables the default subnets to be connected, and cannot selectively connect multiple subnets.</p>"},{"location":"en/advance/with-submariner/#prerequisites","title":"Prerequisites","text":"<ul> <li>The Service CIDRs of the two clusters and the CIDR of the default Subnet cannot overlap.</li> </ul>"},{"location":"en/advance/with-submariner/#install-submariner","title":"Install Submariner","text":"<p>Download the <code>subctl</code> binary and deploy it to the appropriate path:</p> <pre><code>curl -Ls https://get.submariner.io | bash\nexport PATH=$PATH:~/.local/bin\necho export PATH=\\$PATH:~/.local/bin &gt;&gt; ~/.profile\n</code></pre> <p>Change <code>kubeconfig</code> context to the cluster that need to deploy <code>submariner-broker</code>:</p> <pre><code>subctl deploy-broker\n</code></pre> <p>In this document the default subnet CIDR for <code>cluster0</code> is <code>10.16.0.0/16</code> and the join subnet CIDR for <code>cluster0</code> is <code>100.64.0.0/16</code>, the default subnet CIDR for <code>cluster1</code> is <code>11.16.0.0/16</code> and the join subnet CIDR for <code>cluster1</code> is <code>100.68.0.0/16</code>.</p> <p>Switch <code>kubeconfig</code> to <code>cluster0</code> to register the cluster to the broker, and register the gateway node:</p> <pre><code>subctl  join broker-info.subm --clusterid  cluster0 --clustercidr 100.64.0.0/16,10.16.0.0/16  --natt=false --cable-driver vxlan --health-check=false\nkubectl label nodes cluster0 submariner.io/gateway=true\n</code></pre> <p>Switch <code>kubeconfig</code> to <code>cluster1</code> to register the cluster to the broker, and register the gateway node:</p> <pre><code>subctl  join broker-info.subm --clusterid  cluster1 --clustercidr 100.68.0.0/16,11.16.0.0/16  --natt=false --cable-driver vxlan --health-check=false\nkubectl label nodes cluster1 submariner.io/gateway=true\n</code></pre> <p>Next, you can start Pods in each of the two clusters and try to access each other using IPs.</p> <p>Network communication problems can be diagnosed by using the <code>subctl</code> command:</p> <pre><code>subctl show all\nsubctl diagnose all\n</code></pre> <p>For more Submariner operations please read Submariner Usage.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/custom-routes/","title":"Custom Routes","text":"<p>Custom routes can be configured via Pod's annotations. Here is an example:</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: custom-routes\nannotations:\novn.kubernetes.io/routes: |\n[{\n\"dst\": \"192.168.0.101/24\",\n\"gw\": \"10.16.0.254\"\n}, {\n\"gw\": \"10.16.0.254\"\n}]\nspec:\ncontainers:\n- name: nginx\nimage: docker.io/library/nginx:alpine\n</code></pre> <p>Do not set the <code>dst</code> field if you want to configure the default route.</p> <p>For workloads such as Deployment, DaemonSet and StatefulSet, custom routes must be configured via <code>.spec.template.metadata.annotations</code>:</p> <pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nname: custom-routes\nlabels:\napp: nginx\nspec:\nreplicas: 2\nselector:\nmatchLabels:\napp: nginx\ntemplate:\nmetadata:\nlabels:\napp: nginx\nannotations:\novn.kubernetes.io/routes: |\n[{\n\"dst\": \"192.168.0.101/24\",\n\"gw\": \"10.16.0.254\"\n}, {\n\"gw\": \"10.16.0.254\"\n}]\nspec:\ncontainers:\n- name: nginx\nimage: docker.io/library/nginx:alpine\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/dual-stack/","title":"DualStack","text":"<p>Different subnets in Kube-OVN can support different IP protocols. IPv4, IPv6 and dual-stack types of subnets can exist within one cluster. However, it is recommended to use a uniform protocol type within a cluster to simplify usage and maintenance.</p> <p>In order to support dual-stack, the host network needs to meet the dual-stack requirements, and the Kubernetes-related parameters need to be adjusted, please refer to official guide to dual-stack.</p>"},{"location":"en/guide/dual-stack/#create-dual-stack-subnet","title":"Create dual-stack Subnet","text":"<p>When configuring a dual stack Subnet, you only need to set the corresponding subnet CIDR format as <code>cidr=&lt;IPv4 CIDR&gt;,&lt;IPv6 CIDR&gt;</code>.</p> <p>The CIDR order requires IPv4 to come first and IPv6 to come second, as follows.</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata: name: ovn-test\nspec:\ncidrBlock: 10.16.0.0/16,fd00:10:16::/64\nexcludeIps:\n- 10.16.0.1\n- fd00:10:16::1\ngateway: 10.16.0.1,fd00:10:16::1\n</code></pre> <p>If you need to use a dual stack for the default subnet during installation, you need to change the following parameters in the installation script:</p> <pre><code>POD_CIDR=\"10.16.0.0/16,fd00:10:16::/64\"\nJOIN_CIDR=\"100.64.0.0/16,fd00:100:64::/64\"\n</code></pre>"},{"location":"en/guide/dual-stack/#check-pod-address","title":"Check Pod Address","text":"<p>Pods configured for dual-stack networks will be assigned both IPv4 and IPv6 addresses from that subnet, and the results will be displayed in the annotation of the Pod:</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nannotations:\novn.kubernetes.io/allocated: \"true\"\novn.kubernetes.io/cidr: 10.16.0.0/16,fd00:10:16::/64\novn.kubernetes.io/gateway: 10.16.0.1,fd00:10:16::1\novn.kubernetes.io/ip_address: 10.16.0.9,fd00:10:16::9\novn.kubernetes.io/logical_switch: ovn-default\novn.kubernetes.io/mac_address: 00:00:00:14:88:09\novn.kubernetes.io/network_types: geneve\novn.kubernetes.io/routed: \"true\"\n...\npodIP: 10.16.0.9\npodIPs:\n- ip: 10.16.0.9\n- ip: fd00:10:16::9\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/eip-snat/","title":"EIP and SNAT","text":"<p>This configuration is for the network under default VPC, for custom VPC please refer to VPC Gateway</p> <p>Kube-OVN supports SNAT and EIP functionality at the Pod level using the L3 Gateway feature in OVN. By using SNAT, a group of Pods can share an IP address for external access. With the EIP feature, a Pod can be directly associated with an external IP. External services can access the Pod directly through the EIP, and the Pod will also access external services through this EIP.</p> <p></p>"},{"location":"en/guide/eip-snat/#preparation","title":"Preparation","text":"<ul> <li>In order to use the OVN's L3 Gateway capability, a separate NIC must be bridged into the OVS bridge for overlay and underlay network communication.   The host must have other NICs for management.</li> <li>Since packets passing through NAT will go directly to the Underlay network, it is important to confirm that such packets can pass safely on the current network architecture.</li> <li>Currently, there is no conflict detection for EIP and SNAT addresses, and an administrator needs to manually assign them to avoid address conflicts.</li> </ul>"},{"location":"en/guide/eip-snat/#create-config","title":"Create Config","text":"<p>Create ConfigMap <code>ovn-external-gw-config</code> in <code>kube-system</code> Namespace:</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: ovn-external-gw-config\nnamespace: kube-system\ndata:\nenable-external-gw: \"true\"\nexternal-gw-nodes: \"kube-ovn-worker\"\nexternal-gw-nic: \"eth1\"\nexternal-gw-addr: \"172.56.0.1/16\"\nnic-ip: \"172.56.0.254/16\"\nnic-mac: \"16:52:f3:13:6a:25\"\n</code></pre> <ul> <li><code>enable-external-gw</code>: Whether to enable SNAT and EIP functions.</li> <li><code>type</code>: <code>centrailized</code> or <code>distributed</code>\uff0c Default is <code>centralized</code> If <code>distributed</code> is used, all nodes of the cluster need to have the same name NIC to perform the gateway function.</li> <li><code>external-gw-nodes</code>: In <code>centralized</code> mode\uff0cThe names of the node performing the gateway role, comma separated.</li> <li><code>external-gw-nic</code>: The name of the NIC that performs the role of a gateway on the node.</li> <li><code>external-gw-addr</code>: The IP and mask of the physical network gateway.</li> <li><code>nic-ip</code>,<code>nic-mac</code>: The IP and Mac assigned to the logical gateway port needs to be an unoccupied IP and Mac for the physical subnet.</li> </ul>"},{"location":"en/guide/eip-snat/#confirm-the-configuration-take-effect","title":"Confirm the Configuration Take Effect","text":"<p>Check the OVN-NB status to confirm that the <code>ovn-external</code> logical switch exists and that the correct address and chassis are bound to the <code>ovn-cluster-ovn-external</code> logical router port.</p> <pre><code># kubectl ko nbctl show\nswitch 3de4cea7-1a71-43f3-8b62-435a57ef16a6 (ovn-external)\nport ln-ovn-external\n        type: localnet\n        addresses: [\"unknown\"]\nport ovn-external-ovn-cluster\n        type: router\n        router-port: ovn-cluster-ovn-external\nrouter e1eb83ad-34be-4ed5-9a02-fcc8b1d357c4 (ovn-cluster)\nport ovn-cluster-ovn-external\n        mac: \"ac:1f:6b:2d:33:f1\"\nnetworks: [\"172.56.0.100/16\"]\ngateway chassis: [a5682814-2e2c-46dd-9c1c-6803ef0dab66]\n</code></pre> <p>Check the OVS status to confirm that the corresponding NIC is bridged into the <code>br-external</code> bridge:</p> <pre><code># kubectl ko vsctl ${gateway node name} show\ne7d81150-7743-4d6e-9e6f-5c688232e130\n    Bridge br-external\n        Port br-external\n            Interface br-external\n                type: internal\n        Port eno2\n            Interface eno2\n        Port patch-ln-ovn-external-to-br-int\n            Interface patch-ln-ovn-external-to-br-int\n                type: patch\n                options: {peer=patch-br-int-to-ln-ovn-external}\n</code></pre>"},{"location":"en/guide/eip-snat/#config-eip-amd-snat-on-pod","title":"Config EIP amd SNAT on Pod","text":"<p>SNAT and EIP can be configured by adding the <code>ovn.kubernetes.io/snat</code> or <code>ovn.kubernetes.io/eip</code> annotation to the Pod, respectively:</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: pod-gw\nannotations:\novn.kubernetes.io/snat: 172.56.0.200\nspec:\ncontainers:\n- name: snat-pod\nimage: docker.io/library/nginx:alpine\n---\napiVersion: v1\nkind: Pod\nmetadata:\nname: pod-gw\nannotations:\novn.kubernetes.io/eip: 172.56.0.233\nspec:\ncontainers:\n- name: eip-pod\nimage: docker.io/library/nginx:alpine\n</code></pre> <p>The EIP or SNAT rules configured by the Pod can be dynamically adjusted via kubectl or other tools, remember to remove the <code>ovn.kubernetes.io/routed</code> annotation to trigger the routing change.</p> <pre><code>kubectl annotate pod pod-gw ovn.kubernetes.io/eip=172.56.0.221 --overwrite\nkubectl annotate pod pod-gw ovn.kubernetes.io/routed-\n</code></pre> <p>When the EIP or SNAT takes into effect, the <code>ovn.kubernetes.io/routed</code> annotation will be added back.</p>"},{"location":"en/guide/eip-snat/#advanced-configuration","title":"Advanced Configuration","text":"<p>Some args of <code>kube-ovn-controller</code> allow for advanced configuration of SNAT and EIP:</p> <ul> <li><code>--external-gateway-config-ns</code>: The Namespace of Configmap <code>ovn-external-gw-config</code>, default is <code>kube-system</code>\u3002</li> <li><code>--external-gateway-net</code>: The name of the bridge to which the physical NIC is bridged, default is <code>external</code>.</li> <li><code>--external-gateway-vlanid</code>: Physical network Vlan Tag number, default is 0, i.e. no Vlan is used.</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/ippool/","title":"Configure IPPool","text":"<p>IPPool is a more granular IPAM management unit than Subnet. You can subdivide the subnet segment into multiple units through IPPool, and each unit is bound to one or more namespaces.</p>"},{"location":"en/guide/ippool/#instructions","title":"Instructions","text":"<p>Below is an example\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: IPPool\nmetadata:\nname: pool-1\nspec:\nsubnet: ovn-default\nips:\n- \"10.16.0.201\"\n- \"10.16.0.210/30\"\n- \"10.16.0.220..10.16.0.230\"\nnamespaces:\n- ns-1\n</code></pre> <p>Field description:</p> Field Usage Comment subnet Specify the subnet to which it belongs Required ips Specify IP ranges Support three formats: ,  and ... Support IPv6. namespaces Specifies the bound namespaces Optional"},{"location":"en/guide/ippool/#precautions","title":"Precautions","text":"<ol> <li>To ensure compatibility with Workload Universal IP Pool Fixed Address, the name of the IP pool cannot be an IP address;</li> <li>The <code>.spec.ips</code> of the IP pool can specify an IP address beyond the scope of the subnet, but the actual effective IP address is the intersection of <code>.spec.ips</code> and the CIDR of the subnet;</li> <li>Different IP pools of the same subnet cannot contain the same (effective) IP address;</li> <li>The <code>.spec.ips</code> of the IP pool can be modified dynamically;</li> <li>The IP pool will inherit the reserved IP of the subnet. When randomly assigning an IP address from the IP pool, the reserved IP included in the IP pool will be skipped;</li> <li>When randomly assigning an IP address from a subnet, it will only be assigned from a range other than all IP pools in the subnet.</li> </ol> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/loadbalancer-service/","title":"LoadBalancer Type Service","text":"<p>Kube-OVN supports the implementation of VPC and VPC gateway. For specific configurations, please refer to the VPC configuration.</p> <p>Due to the complexity of using VPC gateways, the implementation based on VPC gateways has been simplified. It supports creating LoadBalancer type Services in the default VPC, allowing access to Services in the default VPC through LoadBalancerIP.</p> <p>First, make sure the following conditions are met in the environment:</p> <ol> <li>Install <code>multus-cni</code> and <code>macvlan cni</code>\u3002</li> <li>LoadBalancer Service support relies on simplified implementation of VPC gateway code, still utilizing the vpc-nat-gw image and depending on macvlan for multi-interface functionality support.</li> <li>Currently, it only supports configuration in the default VPC. Support for LoadBalancers in custom VPCs can be referred to in the VPC configuration.</li> </ol>"},{"location":"en/guide/loadbalancer-service/#steps-to-configure-default-vpc-loadbalancer-service","title":"Steps to Configure Default VPC LoadBalancer Service","text":""},{"location":"en/guide/loadbalancer-service/#enable-feature-flag","title":"Enable Feature Flag","text":"<p>Modify the deployment <code>kube-ovn-controller</code> under the kube-system namespace and add the parameter <code>--enable-lb-svc=true</code> to the <code>args</code> section to enable the feature (by default it's set to false).</p> <pre><code>containers:\n- args:\n- /kube-ovn/start-controller.sh\n- --default-cidr=10.16.0.0/16\n- --default-gateway=10.16.0.1\n- --default-gateway-check=true\n- --enable-lb-svc=true                  // parameter is set to true\n</code></pre>"},{"location":"en/guide/loadbalancer-service/#create-networkattachmentdefinition-crd-resource","title":"Create NetworkAttachmentDefinition CRD Resource","text":"<p>Refer to the following YAML and create the <code>net-attach-def</code> resource:</p> <pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: lb-svc-attachment\nnamespace: kube-system\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.0\",\n\"type\": \"macvlan\",\n\"master\": \"eth0\",                            //Physical network card, configure according to the actual situation\n\"mode\": \"bridge\"\n}'\n</code></pre> <p>By default, the physical NIC <code>eth0</code> is used to implement the multi-interface functionality. If another physical NIC is needed, modify the <code>master</code> value to specify the name of the desired physical NIC.</p>"},{"location":"en/guide/loadbalancer-service/#create-subnet","title":"Create Subnet","text":"<p>The created Subnet is used to allocate LoadBalancerIP for the LoadBalancer Service, which should normally  be accessible from outside the cluster. An Underlay Subnet can be configured for address allocation.</p> <p>Refer to the following YAML to create a new subnet:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: attach-subnet\nspec:\nprotocol: IPv4\nprovider: lb-svc-attachment.kube-system          //The provider format is fixed and consists of the Name.Namespace of the net-attach-def resource created in the previous step\ncidrBlock: 172.18.0.0/16\ngateway: 172.18.0.1\nexcludeIps:\n- 172.18.0.0..172.18.0.10\n</code></pre> <p>In the <code>provider</code> parameter of the Subnet, <code>ovn</code> or <code>.ovn</code> suffix is used to indicate that the subnet is managed by Kube-OVN and requires corresponding logical switch records to be created.</p> <p>If <code>provider</code> is neither <code>ovn</code> nor ends with <code>.ovn</code>, Kube-OVN only provides the IPAM functionality to record IP address allocation without handling business logic for the subnet.</p>"},{"location":"en/guide/loadbalancer-service/#create-loadbalancer-service","title":"Create LoadBalancer Service","text":"<p>Refer to the following YAML to create a LoadBalancer Service:</p> <pre><code>apiVersion: v1\nkind: Service\nmetadata:\nannotations:\nlb-svc-attachment.kube-system.kubernetes.io/logical_switch: attach-subnet #Optional\novn.kubernetes.io/attachmentprovider: lb-svc-attachment.kube-system #Required\nlabels:\napp: dynamic\nname: test-service\nnamespace: default\nspec:\nloadBalancerIP: 172.18.0.18 #Optional\nports:\n- name: test\nprotocol: TCP\nport: 80\ntargetPort: 80\nselector:\napp: dynamic\nsessionAffinity: None\ntype: LoadBalancer\n</code></pre> <p>In the yaml, the annotation <code>ovn.kubernetes.io/attachmentprovider</code> is required, and its value is composed of the Name.Namespace of the <code>net-attach-def</code> resource created in the first step. This annotation is used to find the <code>net-attach-def</code> resources when creating Pods.</p> <p>The subnet used for multi-interface address allocation can be specified through an annotation. The annotation key format is <code>net-attach-def</code> resource's <code>Name.Namespace.kubernetes.io/logical_switch</code>. This configuration is <code>optional</code> and if LoadBalancerIP address is not specified, addresses will be dynamically allocated from this subnet and filled into the LoadBalancerIP field.</p> <p>If a static LoadBalancerIP address is required, the <code>spec.loadBalancerIP</code> field can be configured. The address must be within the specified subnet's address range.</p> <p>After creating the Service using the YAML, you can see the Pod startup information in the same namespace as the Service:</p> <pre><code># kubectl get pod\nNAME READY STATUS RESTARTS AGE\nlb-svc-test-service-6869d98dd8-cjvll 1/1 Running 0 107m\n# kubectl get svc\nNAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\ntest-service LoadBalancer 10.109.201.193 172.18.0.18 80:30056/TCP 107m\n</code></pre> <p>When specifying the <code>service.spec.loadBalancerIP</code> parameter, it will be assigned to the service's external IP field. If not specified, the parameter will be assigned a random value.</p> <p>View the YAML output of the test Pod to see the assigned multi-interface addresses:</p> <pre><code># kubectl get pod -o yaml lb-svc-test-service-6869d98dd8-cjvll\napiVersion: v1\nkind: Pod\nmetadata:\n   annotations:\n     k8s.v1.cni.cncf.io/network-status: |-\n       [{\n\"name\": \"kube-ovn\",\n           \"ips\": [\n\"10.16.0.2\"\n],\n           \"default\": true,\n           \"dns\": {}\n},{\n\"name\": \"default/test-service\",\n           \"interface\": \"net1\",\n           \"mac\": \"ba:85:f7:02:9f:42\",\n           \"dns\": {}\n}]\nk8s.v1.cni.cncf.io/networks: default/test-service\n     k8s.v1.cni.cncf.io/networks-status: |-\n       [{\n\"name\": \"kube-ovn\",\n           \"ips\": [\n\"10.16.0.2\"\n],\n           \"default\": true,\n           \"dns\": {}\n},{\n\"name\": \"default/test-service\",\n           \"interface\": \"net1\",\n           \"mac\": \"ba:85:f7:02:9f:42\",\n           \"dns\": {}\n}]\novn.kubernetes.io/allocated: \"true\"\novn.kubernetes.io/cidr: 10.16.0.0/16\n     ovn.kubernetes.io/gateway: 10.16.0.1\n     ovn.kubernetes.io/ip_address: 10.16.0.2\n     ovn.kubernetes.io/logical_router: ovn-cluster\n     ovn.kubernetes.io/logical_switch: ovn-default\n     ovn.kubernetes.io/mac_address: 00:00:00:45:F4:29\n     ovn.kubernetes.io/pod_nic_type: veth-pair\n     ovn.kubernetes.io/routed: \"true\"\ntest-service.default.kubernetes.io/allocated: \"true\"\ntest-service.default.kubernetes.io/cidr: 172.18.0.0/16\n     test-service.default.kubernetes.io/gateway: 172.18.0.1\n     test-service.default.kubernetes.io/ip_address: 172.18.0.18\n     test-service.default.kubernetes.io/logical_switch: attach-subnet\n     test-service.default.kubernetes.io/mac_address: 00:00:00:AF:AA:BF\n     test-service.default.kubernetes.io/pod_nic_type: veth-pair\n</code></pre> <p>Check the service information:</p> <pre><code># kubectl get svc -o yaml test-service\napiVersion: v1\nkind: Service\nmetadata:\n   annotations:\n     kubectl.kubernetes.io/last-applied-configuration: |\n{\"apiVersion\":\"v1\",\"kind\":\"Service\",\"metadata\":{\"annotations\":{\"test-service.default.kubernetes.io/logical_switch\":\"attach-subnet\"},\"labels \":{\"app\":\"dynamic\"},\"name\":\"test-service\",\"namespace\":\"default\"},\"spec\":{\"ports\":[{\"name\":\"test\", \"port\":80,\"protocol\":\"TCP\",\"targetPort\":80}],\"selector\":{\"app\":\"dynamic\"},\"sessionAffinity\":\"None\",\"type\":\"LoadBalancer \"}}\novn.kubernetes.io/vpc:ovn-cluster\n     test-service.default.kubernetes.io/logical_switch: attach-subnet\n   creationTimestamp: \"2022-06-15T09:01:58Z\"\nlabels:\n     app: dynamic\n   name: test-service\n   namespace: default\n   resourceVersion: \"38485\"\nuid: 161edee1-7f6e-40f5-9e09-5a52c44267d0\nspec:\n   allocateLoadBalancerNodePorts: true\nclusterIP: 10.109.201.193\n   clusterIPs:\n   - 10.109.201.193\n   externalTrafficPolicy: Cluster\n   internalTrafficPolicy: Cluster\n   ipFamilies:\n   - IPv4\n   ipFamilyPolicy: SingleStack\n   ports:\n   - name: test\nnodePort: 30056\nport: 80\nprotocol: TCP\n     targetPort: 80\nselector:\n     app: dynamic\n   sessionAffinity: None\n   type: LoadBalancer\nstatus:\n   loadBalancer:\n     ingress:\n     - ip: 172.18.0.18\n</code></pre>"},{"location":"en/guide/loadbalancer-service/#testing-loadbalancerip-access","title":"Testing LoadBalancerIP access","text":"<p>Refer to the following YAML to create a test Pod that serves as the Endpoints for the Service:</p> <pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nlabels:\napp: dynamic\nname: dynamic\nnamespace: default\nspec:\nreplicas: 2\nselector:\nmatchLabels:\napp: dynamic\nstrategy:\nrollingUpdate:\nmaxSurge: 25%\nmaxUnavailable: 25%\ntype: RollingUpdate\ntemplate:\nmetadata:\ncreationTimestamp: null\nlabels:\napp: dynamic\nspec:\ncontainers:\n- image: docker.io/library/nginx:alpine\nimagePullPolicy: IfNotPresent\nname: nginx\ndnsPolicy: ClusterFirst\nrestartPolicy: Always\n</code></pre> <p>Under normal circumstances, the provided subnet addresses should be accessible from outside the cluster. To verify, access the Service's <code>LoadBalancerIP:Port</code> from within the cluster and check if the access is successful.</p> <pre><code># curl 172.18.0.11:80\n&lt;html&gt;\n&lt;head&gt;\n        &lt;title&gt;Hello World!&lt;/title&gt;\n        &lt;link href='//fonts.googleapis.com/css?family=Open+Sans:400,700' rel='stylesheet' type='text/css'&gt;\n        &lt;style&gt;\n        body {\nbackground-color: white;\ntext-align: center;\npadding: 50px;\nfont-family: \"Open Sans\",\"Helvetica Neue\",Helvetica,Arial,sans-serif;\n}\n#logo {\nmargin-bottom: 40px;\n}\n&lt;/style&gt;\n&lt;/head&gt;\n&lt;body&gt;\n                &lt;h1&gt;Hello World!&lt;/h1&gt;\n                                &lt;h3&gt;Links found&lt;/h3&gt;\n        &lt;h3&gt;I am on  dynamic-7d8d7874f5-hsgc4&lt;/h3&gt;\n        &lt;h3&gt;Cookie                  =&lt;/h3&gt;\n                                        &lt;b&gt;KUBERNETES&lt;/b&gt; listening in 443 available at tcp://10.96.0.1:443&lt;br /&gt;\n                                                &lt;h3&gt;my name is hanhouchao!&lt;/h3&gt;\n                        &lt;h3&gt; RequestURI='/'&lt;/h3&gt;\n&lt;/body&gt;\n&lt;/html&gt;\n</code></pre> <p>Enter the Pod created by the Service and check the network information:</p> <pre><code># ip a\n4: net1@if62: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1500 qdisc noqueue state UP group default\n    link/ether ba:85:f7:02:9f:42 brd ff:ff:ff:ff:ff:ff link-netnsid 0\ninet 172.18.0.18/16 scope global net1\n       valid_lft forever preferred_lft forever\n    inet6 fe80::b885:f7ff:fe02:9f42/64 scope link\n       valid_lft forever preferred_lft forever\n36: eth0@if37: &lt;BROADCAST,MULTICAST,UP,LOWER_UP&gt; mtu 1400 qdisc noqueue state UP group default\n    link/ether 00:00:00:45:f4:29 brd ff:ff:ff:ff:ff:ff link-netnsid 0\ninet 10.16.0.2/16 brd 10.16.255.255 scope global eth0\n       valid_lft forever preferred_lft forever\n    inet6 fe80::200:ff:fe45:f429/64 scope link\n       valid_lft forever preferred_lft forever\n\n# ip rule\n0: from all lookup local\n32764: from all iif eth0 lookup 100\n32765: from all iif net1 lookup 100\n32766: from all lookup main\n32767: from all lookup default\n\n# ip route show table 100\ndefault via 172.18.0.1 dev net1\n10.109.201.193 via 10.16.0.1 dev eth0\n172.18.0.0/16 dev net1 scope link\n\n# iptables -t nat -L -n -v\nChain PREROUTING (policy ACCEPT 0 packets, 0 bytes)\npkts bytes target     prot opt in     out     source               destination\n    0     0 DNAT       tcp  --  *      *       0.0.0.0/0            172.18.0.18          tcp dpt:80 to:10.109.201.193:80\n\nChain INPUT (policy ACCEPT 0 packets, 0 bytes)\npkts bytes target     prot opt in     out     source               destination\n\nChain OUTPUT (policy ACCEPT 0 packets, 0 bytes)\npkts bytes target     prot opt in     out     source               destination\n\nChain POSTROUTING (policy ACCEPT 0 packets, 0 bytes)\npkts bytes target     prot opt in     out     source               destination\n    0     0 MASQUERADE  all  --  *      *       0.0.0.0/0            10.109.201.193\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/mirror/","title":"Traffic Mirror","text":"<p>The traffic mirroring feature allows packets to and from the container network to be copied to a specific NIC of the host. Administrators or developers can listen to this NIC to get the complete container network traffic for further analysis, monitoring, security auditing and other operations. It can also be integrated with traditional NPM for more fine-grained traffic visibility.</p> <p>The traffic mirroring feature introduces some performance loss, with an additional CPU consumption of 5% to 10% depending on CPU performance and traffic characteristics.</p> <p></p>"},{"location":"en/guide/mirror/#global-traffic-mirroring-settings","title":"Global Traffic Mirroring Settings","text":"<p>The traffic mirroring is disabled by default, please modify the args of <code>kube-ovn-cni</code> DaemonSet to enable it:</p> <ul> <li><code>--enable-mirror=true</code>: Whether to enable traffic mirroring.</li> <li><code>--mirror-iface=mirror0</code>: The name of the NIC that the traffic mirror is copied to. This NIC can be a physical NIC that already exists on the host machine.   At this point the NIC will be bridged into the br-int bridge and the mirrored traffic will go directly to the underlying switch.   If the NIC name does not exist, Kube-OVN will automatically create a virtual NIC with the same name, through which the administrator or developer can access all traffic on the current node on the host.   The default is <code>mirror0</code>.</li> </ul> <p>Next, you can listen to the traffic on <code>mirror0</code> with tcpdump or other traffic analysis tools.</p> <pre><code>tcpdump -ni mirror0\n</code></pre>"},{"location":"en/guide/mirror/#pod-level-mirroring-settings","title":"Pod Level Mirroring Settings","text":"<p>If you only need to mirror some Pod traffic, you need to disable the global traffic mirroring and then add the <code>ovn.kubernetes.io/mirror</code> annotation on a specific Pod to enable Pod-level traffic mirroring.</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: mirror-pod\nnamespace: ls1\nannotations:\novn.kubernetes.io/mirror: \"true\"\nspec:\ncontainers:\n- name: mirror-pod\nimage: docker.io/library/nginx:alpine\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/networkpolicy-log/","title":"NetworkPolicy Logging","text":"<p>NetworkPolicy is a interface provided by Kubernetes and implemented by Kube-OVN through OVN's ACLs. With NetworkPolicy, if the networks are down, it is difficult to determine whether it is caused by a network failure or a NetworkPolicy rule problem. Kube-OVN provides NetworkPolicy logging to help administrators quickly locate whether a NetworkPolicy drop rule has been hit, and to record the illegal accesses.</p> <p>Once NetworkPolicy logging is turned on, logs need to be printed for every packet that hits a Drop rule, which introduces additional performance overhead. Under a malicious attack, a large number of logs in a short period of time may exhaust the CPU. We recommend turning off logging by default in production environments and dynamically turning it on when you need to troubleshoot problems.</p>"},{"location":"en/guide/networkpolicy-log/#enable-networkpolicy-logging","title":"Enable NetworkPolicy Logging","text":"<p>Add the annotation <code>ovn.kubernetes.io/enable_log</code> to the NetworkPolicy where logging needs to be enabled, as follows:</p> <pre><code>apiVersion: networking.k8s.io/v1\nkind: NetworkPolicy\nmetadata:\nname: default-deny-ingress\nnamespace: kube-system\nannotations:\novn.kubernetes.io/enable_log: \"true\"\nspec:\npodSelector: {}\npolicyTypes:\n- Ingress\n</code></pre> <p>Next, you can observe the log of dropped packets in <code>/var/log/ovn/ovn-controller.log</code> on the host of the corresponding Pod:</p> <pre><code># tail -f /var/log/ovn/ovn-controller.log\n2022-07-20T05:55:03.229Z|00394|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: udp,vlan_tci=0x0000,dl_src=00:00:00:21:b7:d1,dl_dst=00:00:00:8d:0b:86,nw_src=10.16.0.10,nw_dst=10.16.0.7,nw_tos=0,nw_ecn=0,nw_ttl=63,tp_src=54343,tp_dst=53\n2022-07-20T05:55:06.229Z|00395|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: udp,vlan_tci=0x0000,dl_src=00:00:00:21:b7:d1,dl_dst=00:00:00:8d:0b:86,nw_src=10.16.0.9,nw_dst=10.16.0.7,nw_tos=0,nw_ecn=0,nw_ttl=63,tp_src=44187,tp_dst=53\n2022-07-20T05:55:08.230Z|00396|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: udp,vlan_tci=0x0000,dl_src=00:00:00:21:b7:d1,dl_dst=00:00:00:8d:0b:86,nw_src=10.16.0.10,nw_dst=10.16.0.7,nw_tos=0,nw_ecn=0,nw_ttl=63,tp_src=54274,tp_dst=53\n2022-07-20T05:55:11.231Z|00397|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: udp,vlan_tci=0x0000,dl_src=00:00:00:21:b7:d1,dl_dst=00:00:00:8d:0b:86,nw_src=10.16.0.9,nw_dst=10.16.0.7,nw_tos=0,nw_ecn=0,nw_ttl=63,tp_src=32778,tp_dst=53\n2022-07-20T05:55:11.231Z|00398|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: udp,vlan_tci=0x0000,dl_src=00:00:00:21:b7:d1,dl_dst=00:00:00:8d:0b:86,nw_src=10.16.0.9,nw_dst=10.16.0.7,nw_tos=0,nw_ecn=0,nw_ttl=63,tp_src=34188,tp_dst=53\n2022-07-20T05:55:13.231Z|00399|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: udp,vlan_tci=0x0000,dl_src=00:00:00:21:b7:d1,dl_dst=00:00:00:8d:0b:86,nw_src=10.16.0.10,nw_dst=10.16.0.7,nw_tos=0,nw_ecn=0,nw_ttl=63,tp_src=43290,tp_dst=53\n2022-07-20T05:55:22.096Z|00400|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: icmp,vlan_tci=0x0000,dl_src=00:00:00:6c:42:91,dl_dst=00:00:00:a5:d7:63,nw_src=10.16.0.9,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0\n2022-07-20T05:55:22.097Z|00401|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: icmp,vlan_tci=0x0000,dl_src=00:00:00:6c:42:91,dl_dst=00:00:00:a5:d7:63,nw_src=10.16.0.9,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0\n2022-07-20T05:55:22.098Z|00402|acl_log(ovn_pinctrl0)|INFO|name=\"&lt;unnamed&gt;\", verdict=drop, severity=warning, direction=to-lport: icmp,vlan_tci=0x0000,dl_src=00:00:00:6c:42:91,dl_dst=00:00:00:a5:d7:63,nw_src=10.16.0.9,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0\n</code></pre>"},{"location":"en/guide/networkpolicy-log/#disable-networkpolicy-logging","title":"Disable NetworkPolicy Logging","text":"<p>Set annotation <code>ovn.kubernetes.io/enable_log</code> in the corresponding NetworkPolicy to <code>false</code> to disable NetworkPolicy logging:</p> <pre><code>kubectl annotate networkpolicy -n kube-system default-deny-ingress ovn.kubernetes.io/enable_log=false --overwrite\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/prometheus-grafana/","title":"Monitor and Dashboard","text":"<p>Kube-OVN can export network control plane information and network data plane quality information metrics to the external in formats supported by Prometheus.</p> <p>We use the CRD provided by kube-prometheus to define the corresponding Prometheus monitoring rules. For all monitoring metrics supported by Kube-OVN, please refer to Kube-OVN Monitoring Metrics.</p> <p>If you are using native Prometheus, please refer to Configuring Native Prometheus for configuration.</p>"},{"location":"en/guide/prometheus-grafana/#install-prometheus-monitor","title":"Install Prometheus Monitor","text":"<p>Kube-OVN uses Prometheus Monitor CRD to manage the monitoring output.</p> <pre><code># network quality related monitoring metrics\nkubectl apply -f https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/pinger-monitor.yaml\n# kube-ovn-controller metrics\nkubectl apply -f https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/controller-monitor.yaml\n# kube-ovn-cni metrics\nkubectl apply -f https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/cni-monitor.yaml\n# ovn metrics\nkubectl apply -f https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/ovn-monitor.yaml\n</code></pre> <p>The default interval for Prometheus pull is 15s, if you need to adjust it, modify the <code>interval</code> value in yaml.</p>"},{"location":"en/guide/prometheus-grafana/#import-grafana-dashboard","title":"Import Grafana Dashboard","text":"<p>Kube-OVN provides a predefined Grafana Dashboard to display control plane and data plane related metrics.</p> <p>Download the corresponding Dashboard template:</p> <pre><code># network quality related monitoring dashboard\nwget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/pinger-grafana.json\n# kube-ovn-controller dashboard\nwget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/controller-grafana.json\n# kube-ovn-cni dashboard\nwget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/cni-grafana.json\n# ovn dashboard\nwget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/ovn-grafana.json\n# ovs dashboard\nwget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/monitoring/ovs-grafana.json\n</code></pre> <p>Import these templates into Grafana and set the data source to the corresponding Prometheus to see the following Dashboards.</p> <p><code>kube-ovn-controller</code> dashboard:</p> <p></p> <p><code>kube-ovn-pinger</code> dashboard:</p> <p></p> <p><code>kube-ovn-cni</code> dashboard:</p> <p></p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/prometheus/","title":"Config Native Prometheus","text":"<p>Kube-OVN provides rich monitoring data for OVN/OVS health status checks and connectivity checks of container and host networks, and Kube-OVN is configured with ServiceMonitor for Prometheus to dynamically obtain monitoring metrics.</p> <p>In some cases, where only Prometheus Server is installed and no other components are installed, you can dynamically obtain monitoring data for the cluster environment by modifying the configuration of Prometheus.</p>"},{"location":"en/guide/prometheus/#config-prometheus","title":"Config Prometheus","text":"<p>The following configuration documentation, referenced from Prometheus Service Discovery.</p>"},{"location":"en/guide/prometheus/#permission-configuration","title":"Permission Configuration","text":"<p>Prometheus is deployed in the cluster and needs to access the k8s apiserver to query the monitoring data of the containers.</p> <p>Refer to the following yaml to configure the permissions required by Prometheus:</p> <pre><code>apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\nname: prometheus\nrules:\n- apiGroups: [\"\"]\nresources:\n- nodes\n- nodes/proxy\n- services\n- endpoints\n- pods\nverbs: [\"get\", \"list\", \"watch\"]\n- apiGroups:\n- extensions\nresources:\n- ingresses\nverbs: [\"get\", \"list\", \"watch\"]\n- nonResourceURLs: [\"/metrics\"]\nverbs: [\"get\"]\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\nname: prometheus\nnamespace: default\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\nname: prometheus\nroleRef:\napiGroup: rbac.authorization.k8s.io\nkind: ClusterRole\nname: prometheus\nsubjects:\n- kind: ServiceAccount\nname: prometheus\nnamespace: default\n</code></pre>"},{"location":"en/guide/prometheus/#prometheus-configmap","title":"Prometheus ConfigMap","text":"<p>The startup of Prometheus relies on the configuration file prometheus.yml, the contents of which can be configured in ConfigMap and dynamically mounted to the Pod.</p> <p>Create the ConfigMap file used by Prometheus by referring to the following yaml:</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: prometheus-config\ndata:\nprometheus.yml: |-\nglobal:\nscrape_interval:     15s \nevaluation_interval: 15s\nscrape_configs:\n- job_name: 'prometheus'\nstatic_configs:\n- targets: ['localhost:9090']\n\n- job_name: 'kubernetes-nodes'\ntls_config:\nca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\nbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\nkubernetes_sd_configs:\n- role: node\n\n- job_name: 'kubernetes-service'\ntls_config:\nca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\nbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\nkubernetes_sd_configs:\n- role: service\n\n- job_name: 'kubernetes-endpoints'\ntls_config:\nca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\nbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\nkubernetes_sd_configs:\n- role: endpoints\n\n- job_name: 'kubernetes-ingress'\ntls_config:\nca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\nbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\nkubernetes_sd_configs:\n- role: ingress\n\n- job_name: 'kubernetes-pods'\ntls_config:\nca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\nbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\nkubernetes_sd_configs:\n- role: pod\n</code></pre> <p>Prometheus provides role-based querying of Kubernetes resource monitoring operations, which can be configured in the official documentation kubernetes_sd_config\u3002</p>"},{"location":"en/guide/prometheus/#deploy-prometheus","title":"Deploy Prometheus","text":"<p>Deploy Prometheus Server by referring to the following yaml:</p> <pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nlabels:\napp: prometheus\nname: prometheus\nnamespace: default\nspec:\nreplicas: 1\nselector:\nmatchLabels:\napp: prometheus\nstrategy:\nrollingUpdate:\nmaxSurge: 25%\nmaxUnavailable: 25%\ntype: RollingUpdate\ntemplate:\nmetadata:\nlabels:\napp: prometheus\nspec:\nserviceAccountName: prometheus\nserviceAccount: prometheus\ncontainers:\n- image: docker.io/prom/prometheus:latest\nimagePullPolicy: IfNotPresent\nname: prometheus\ncommand:\n- \"/bin/prometheus\"\nargs:\n- \"--config.file=/etc/prometheus/prometheus.yml\"\nports:\n- containerPort: 9090\nprotocol: TCP\nvolumeMounts:\n- mountPath: \"/etc/prometheus\"\nname: prometheus-config\nvolumes:\n- name: prometheus-config\nconfigMap:\nname: prometheus-config\n</code></pre> <p>Deploy Prometheus Service by referring to the following yaml:</p> <pre><code>kind: Service\napiVersion: v1\nmetadata:\nname: prometheus\nnamespace: default\nlabels:\nname: prometheus\nspec:\nports:\n- name: test\nprotocol: TCP\nport: 9090\ntargetPort: 9090\ntype: NodePort\nselector:\napp: prometheus\nsessionAffinity: None\n</code></pre> <p>After exposing Prometheus through NodePort, Prometheus can be accessed through the node address.</p>"},{"location":"en/guide/prometheus/#prometheus-metrics-config","title":"Prometheus Metrics Config","text":"<p>View information about Prometheus on the environment:</p> <pre><code># kubectl get svc \nNAME         TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)          AGE\nkubernetes   ClusterIP   10.4.0.1       &lt;none&gt;        443/TCP          8d\nprometheus   NodePort    10.4.102.222   &lt;none&gt;        9090:32611/TCP   8d\n# kubectl get pod -o wide\nNAME                          READY   STATUS    RESTARTS   AGE    IP          NODE              NOMINATED NODE   READINESS GATES\nprometheus-7544b6b84d-v9m8s   1/1     Running   0          3d5h   10.3.0.7    192.168.137.219   &lt;none&gt;           &lt;none&gt;\n# kubectl get endpoints -o wide\nNAME         ENDPOINTS                                                        AGE\nkubernetes   192.168.136.228:6443,192.168.136.232:6443,192.168.137.219:6443   8d\nprometheus   10.3.0.7:9090                                                    8d\n</code></pre> <p>Access Prometheus via NodePort to see the data dynamically queried by Status/Service Discovery:</p> <p></p> <p>You can see that you can currently query all the service data information on the cluster.</p>"},{"location":"en/guide/prometheus/#configure-to-query-specified-resource","title":"Configure to Query Specified Resource","text":"<p>The ConfigMap configuration above queries all resource data. If you only need resource data for a certain role, you can add filter conditions.</p> <p>Take Service as an example, modify the ConfigMap content to query only the service monitoring data:</p> <pre><code>    - job_name: 'kubernetes-service'\ntls_config:\nca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\nbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token\nkubernetes_sd_configs:\n- role: service\nrelabel_configs:\n- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]\naction: \"keep\"\nregex: \"true\"\n- action: labelmap\nregex: __meta_kubernetes_service_label_(.+)\n- source_labels: [__meta_kubernetes_namespace]\ntarget_label: kubernetes_namespace\n- source_labels: [__meta_kubernetes_service_name]\ntarget_label: kubernetes_service_name\n- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]\naction: replace\ntarget_label: __metrics_path__\nregex: \"(.+)\"\n</code></pre> <p>Check the Kube-OVN Service in kube-system Namespace:</p> <pre><code># kubectl get svc -n kube-system\nNAME                  TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)                  AGE\nkube-dns              ClusterIP   10.4.0.10      &lt;none&gt;        53/UDP,53/TCP,9153/TCP   13d\nkube-ovn-cni          ClusterIP   10.4.228.60    &lt;none&gt;        10665/TCP                13d\nkube-ovn-controller   ClusterIP   10.4.172.213   &lt;none&gt;        10660/TCP                13d\nkube-ovn-monitor      ClusterIP   10.4.242.9     &lt;none&gt;        10661/TCP                13d\nkube-ovn-pinger       ClusterIP   10.4.122.52    &lt;none&gt;        8080/TCP                 13d\novn-nb                ClusterIP   10.4.80.213    &lt;none&gt;        6641/TCP                 13d\novn-northd            ClusterIP   10.4.126.234   &lt;none&gt;        6643/TCP                 13d\novn-sb                ClusterIP   10.4.216.249   &lt;none&gt;        6642/TCP                 13d\n</code></pre> <p>Add annotation <code>prometheus.io/scrape=\"true\"</code> to Service\uff1a</p> <pre><code># kubectl annotate svc -n kube-system kube-ovn-cni  prometheus.io/scrape=true\nservice/kube-ovn-cni annotated\n# kubectl annotate svc -n kube-system kube-ovn-controller  prometheus.io/scrape=true\nservice/kube-ovn-controller annotated\n# kubectl annotate svc -n kube-system kube-ovn-monitor  prometheus.io/scrape=true\nservice/kube-ovn-monitor annotated\n# kubectl annotate svc -n kube-system kube-ovn-pinger  prometheus.io/scrape=true\nservice/kube-ovn-pinger annotated\n</code></pre> <p>Check the configured Service information:</p> <pre><code># kubectl get svc -o yaml -n kube-system kube-ovn-controller\napiVersion: v1\nkind: Service\nmetadata:\n  annotations:\n    helm.sh/chart-version: v3.10.0-alpha.55\n    helm.sh/original-name: kube-ovn-controller\n    ovn.kubernetes.io/vpc: ovn-cluster\n    prometheus.io/scrape: \"true\"                        // added annotation\n  labels:\n    app: kube-ovn-controller\n  name: kube-ovn-controller\n  namespace: kube-system\nspec:\n  clusterIP: 10.4.172.213\n  clusterIPs:\n  - 10.4.172.213\n  internalTrafficPolicy: Cluster\n  ipFamilies:\n  - IPv4\n  ipFamilyPolicy: SingleStack\n  ports:\n  - name: metrics\n    port: 10660\nprotocol: TCP\n    targetPort: 10660\nselector:\n    app: kube-ovn-controller\n  sessionAffinity: None\n  type: ClusterIP\nstatus:\n  loadBalancer: {}\n</code></pre> <p>Looking at the Prometheus Status Targets information, you can only see the Services with annotation:</p> <p></p> <p>For more information about adding filter parameters to relabel, please check Prometheus-Relabel\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/qos/","title":"Manage QoS","text":"<p>Kube-OVN supports two types of Pod level QoS:</p> <ul> <li>Maximum bandwidth limit QoS.</li> <li><code>linux-netem</code>, QoS for simulating latency and packet loss that can be used for simulation testing.</li> </ul> <p>Currently, only Pod level QoS is supported, and QoS restrictions at the Namespace or Subnet level are not supported.</p>"},{"location":"en/guide/qos/#maximum-bandwidth-limit-qos","title":"Maximum Bandwidth Limit QoS","text":"<p>This type of QoS can be dynamically configured via Pod annotation and can be adjusted without restarting running Pod. Bandwidth speed limit unit is <code>Mbit/s</code>.</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nname: qos\nnamespace: ls1\nannotations:\novn.kubernetes.io/ingress_rate: \"3\"\novn.kubernetes.io/egress_rate: \"1\"\nspec:\ncontainers:\n- name: qos\nimage: docker.io/library/nginx:alpine\n</code></pre> <p>Use annotation to dynamically adjust QoS:</p> <pre><code>kubectl annotate --overwrite  pod nginx-74d5899f46-d7qkn ovn.kubernetes.io/ingress_rate=3\n</code></pre>"},{"location":"en/guide/qos/#test-qos","title":"Test QoS","text":"<p>Deploy the containers needed for performance testing:</p> <pre><code>kind: DaemonSet\napiVersion: apps/v1\nmetadata:\nname: perf\nnamespace: ls1\nlabels:\napp: perf\nspec:\nselector:\nmatchLabels:\napp: perf\ntemplate:\nmetadata:\nlabels:\napp: perf\nspec:\ncontainers:\n- name: nginx\nimage: docker.io/kubeovn/perf\n</code></pre> <p>Exec into one Pod and run iperf3 server:</p> <pre><code># kubectl exec -it perf-4n4gt -n ls1 sh\n# iperf3 -s\n-----------------------------------------------------------\nServer listening on 5201\n-----------------------------------------------------------\n</code></pre> <p>Exec into the other Pod and run iperf3 client to connect above server address:</p> <pre><code># kubectl exec -it perf-d4mqc -n ls1 sh\n# iperf3 -c 10.66.0.12\nConnecting to host 10.66.0.12, port 5201\n[  4] local 10.66.0.14 port 51544 connected to 10.66.0.12 port 5201\n[ ID] Interval           Transfer     Bandwidth       Retr  Cwnd\n[  4]   0.00-1.00   sec  86.4 MBytes   725 Mbits/sec    3    350 KBytes\n[  4]   1.00-2.00   sec  89.9 MBytes   754 Mbits/sec  118    473 KBytes\n[  4]   2.00-3.00   sec   101 MBytes   848 Mbits/sec  184    586 KBytes\n[  4]   3.00-4.00   sec   104 MBytes   875 Mbits/sec  217    671 KBytes\n[  4]   4.00-5.00   sec   111 MBytes   935 Mbits/sec  175    772 KBytes\n[  4]   5.00-6.00   sec   100 MBytes   840 Mbits/sec  658    598 KBytes\n[  4]   6.00-7.00   sec   106 MBytes   890 Mbits/sec  742    668 KBytes\n[  4]   7.00-8.00   sec   102 MBytes   857 Mbits/sec  764    724 KBytes\n[  4]   8.00-9.00   sec  97.4 MBytes   817 Mbits/sec  1175    764 KBytes\n[  4]   9.00-10.00  sec   111 MBytes   934 Mbits/sec  1083    838 KBytes\n- - - - - - - - - - - - - - - - - - - - - - - - -\n[ ID] Interval           Transfer     Bandwidth       Retr\n[  4]   0.00-10.00  sec  1010 MBytes   848 Mbits/sec  5119             sender\n[  4]   0.00-10.00  sec  1008 MBytes   846 Mbits/sec                  receiver\n\niperf Done.\n</code></pre> <p>Modify the ingress bandwidth QoS for the first Pod:</p> <pre><code>kubectl annotate --overwrite  pod perf-4n4gt -n ls1 ovn.kubernetes.io/ingress_rate=30\n</code></pre> <p>Test the Pod bandwidth again from the second Pod:</p> <pre><code># iperf3 -c 10.66.0.12\nConnecting to host 10.66.0.12, port 5201\n[  4] local 10.66.0.14 port 52372 connected to 10.66.0.12 port 5201\n[ ID] Interval           Transfer     Bandwidth       Retr  Cwnd\n[  4]   0.00-1.00   sec  3.66 MBytes  30.7 Mbits/sec    2   76.1 KBytes\n[  4]   1.00-2.00   sec  3.43 MBytes  28.8 Mbits/sec    0    104 KBytes\n[  4]   2.00-3.00   sec  3.50 MBytes  29.4 Mbits/sec    0    126 KBytes\n[  4]   3.00-4.00   sec  3.50 MBytes  29.3 Mbits/sec    0    144 KBytes\n[  4]   4.00-5.00   sec  3.43 MBytes  28.8 Mbits/sec    0    160 KBytes\n[  4]   5.00-6.00   sec  3.43 MBytes  28.8 Mbits/sec    0    175 KBytes\n[  4]   6.00-7.00   sec  3.50 MBytes  29.3 Mbits/sec    0    212 KBytes\n[  4]   7.00-8.00   sec  3.68 MBytes  30.9 Mbits/sec    0    294 KBytes\n[  4]   8.00-9.00   sec  3.74 MBytes  31.4 Mbits/sec    0    398 KBytes\n[  4]   9.00-10.00  sec  3.80 MBytes  31.9 Mbits/sec    0    526 KBytes\n- - - - - - - - - - - - - - - - - - - - - - - - -\n[ ID] Interval           Transfer     Bandwidth       Retr\n[  4]   0.00-10.00  sec  35.7 MBytes  29.9 Mbits/sec    2             sender\n[  4]   0.00-10.00  sec  34.5 MBytes  29.0 Mbits/sec                  receiver\n\niperf Done.\n</code></pre>"},{"location":"en/guide/qos/#linux-netem-qos","title":"linux-netem QoS","text":"<p>Pod can use annotation below to config <code>linux-netem</code> type QoS\uff1a <code>ovn.kubernetes.io/latency</code>\u3001<code>ovn.kubernetes.io/limit</code> and <code>ovn.kubernetes.io/loss</code>\u3002</p> <ul> <li><code>ovn.kubernetes.io/latency</code>: Set the Pod traffic delay to an integer value in ms.</li> <li><code>ovn.kubernetes.io/limit</code>\uff1a Set the maximum number of packets that the <code>qdisc</code> queue can hold, and takes an integer value, such as 1000.</li> <li><code>ovn.kubernetes.io/loss</code>\uff1a Set packet loss probability, the value is float type, for example, the value is 20, then it is set 20% packet loss probability.</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/setup-options/","title":"Installation and Configuration Options","text":"<p>In One-Click Installation we use the default configuration for installation. Kube-OVN also supports more custom configurations, which can be configured in the installation script, or later by changing the parameters of individual components. This document will describe what these customization options do, and how to configure them.</p>"},{"location":"en/guide/setup-options/#built-in-network-settings","title":"Built-in Network Settings","text":"<p>Kube-OVN will configure two built-in Subnets during installation:</p> <ol> <li><code>default</code> Subnet, as the default subnet used by the Pod to assign IPs, with a default CIDR of <code>10.16.0.0/16</code> and a gateway of <code>10.16.0.1</code>.</li> <li>The <code>join</code> subnet, as a special subnet for network communication between the Node and Pod, has a default CIDR of <code>100.64.0.0/16</code> and a gateway of <code>100.64.0.1</code>.</li> </ol> <p>The configuration of these two subnets can be changed during installation via the installation scripts variables:</p> <pre><code>POD_CIDR=\"10.16.0.0/16\"\nPOD_GATEWAY=\"10.16.0.1\"\nJOIN_CIDR=\"100.64.0.0/16\"\nEXCLUDE_IPS=\"\"\n</code></pre> <p><code>EXCLUDE_IP</code> sets the address range for which <code>kube-ovn-controller</code> will not automatically assign from it, the format is: <code>192.168.10.20..192.168.10.30</code>.</p> <p>Note that in the Overlay case these two Subnets CIDRs cannot conflict with existing host networks and Service CIDRs.</p> <p>You can change the address range of both Subnets after installation by referring to Change Subnet CIDR and Change Join Subnet CIDR.</p>"},{"location":"en/guide/setup-options/#config-service-cidr","title":"Config Service CIDR","text":"<p>Since some of the iptables and routing rules set by <code>kube-proxy</code> will conflict with the rules set by Kube-OVN, Kube-OVN needs to know the CIDR of the service to set the corresponding rules correctly.</p> <p>This can be done by modifying the installation script:</p> <pre><code>SVC_CIDR=\"10.96.0.0/12\"  </code></pre> <p>You can also modify the args of the <code>kube-ovn-controller</code> Deployment after installation:</p> <pre><code>args:\n- --service-cluster-ip-range=10.96.0.0/12\n</code></pre>"},{"location":"en/guide/setup-options/#overlay-nic-selection","title":"Overlay NIC Selection","text":"<p>In the case of multiple NICs on a node, Kube-OVN will select the NIC corresponding to the Kubernetes Node IP as the NIC for cross-node communication between containers and establish the corresponding tunnel.</p> <p>If you need to select another NIC to create a container tunnel, you can change it in the installation script:</p> <pre><code>IFACE=eth1\n</code></pre> <p>This option supports regular expressions separated by commas, e.g. 'ens[a-z0-9],eth[a-z0-9]'.</p> <p>It can also be adjusted after installation by modifying the args of the <code>kube-ovn-cni</code> DaemonSet:</p> <pre><code>args:\n- --iface=eth1\n</code></pre> <p>If each machine has a different NIC name and there is no fixed pattern, you can use the node annotation <code>ovn.kubernetes.io/tunnel_interface</code> to configure each node one by one. This annotation will override the configuration of <code>iface</code>.</p> <pre><code>kubectl annotate node no1 ovn.kubernetes.io/tunnel_interface=ethx\n</code></pre>"},{"location":"en/guide/setup-options/#config-mtu","title":"Config MTU","text":"<p>Since Overlay encapsulation requires additional space, Kube-OVN will adjust the MTU of the container NIC based on the MTU of the selected NIC when creating the container NIC. By default, the Pod NIC MTU is the host NIC MTU - 100 on the Overlay Subnet, and the Pod NIC and host NIC have the same MTU on the Underlay Subnet.</p> <p>If you need to adjust the size of the MTU under the Overlay subnet, you can modify the parameters of the <code>kube-ovn-cni</code> DaemonSet:</p> <pre><code>args:\n- --mtu=1333\n</code></pre>"},{"location":"en/guide/setup-options/#global-traffic-mirroring-setting","title":"Global Traffic Mirroring Setting","text":"<p>When global traffic mirroring is enabled, Kube-OVN will create a <code>mirror0</code> virtual NIC on each node and copy all container network traffic from the current machine to that NIC\uff0c Users can perform traffic analysis with tcpdump and other tools. This function can be enabled in the installation script:</p> <pre><code>ENABLE_MIRROR=true\n</code></pre> <p>It can also be adjusted after installation by modifying the args of the <code>kube-ovn-cni</code> DaemonSet:</p> <pre><code>args:\n- --enable-mirror=true\n</code></pre> <p>The ability to mirror traffic is disabled in the default installation, if you need fine-grained traffic mirroring or need to mirror traffic to additional NICs please refer to Traffic Mirror.</p>"},{"location":"en/guide/setup-options/#lb-settings","title":"LB Settings","text":"<p>Kube-OVN uses L2 LB in OVN to implement service forwarding. In Overlay scenarios, users can choose to use <code>kube-proxy</code> for service traffic forwarding, in which case the LB function of Kube-OVN can be disabled to achieve better performance on the control plane and data plane.</p> <p>This feature can be configured in the installation script:</p> <pre><code>ENABLE_LB=false\n</code></pre> <p>It can also be configured after installation by changing the args of the <code>kube-ovn-controller</code> Deployment:</p> <pre><code>args:\n- --enable-lb=false\n</code></pre> <p>The LB feature is enabled in the default installation.</p> <p>The spec field <code>enableLb</code> has been added to the subnet crd definition since Kube-OVN v1.12.0 to migrate the LB function of Kube-OVN to the subnet level. You can set whether to enable the LB function based on different subnets. The <code>enable-lb</code> parameter in the <code>kube-ovn-controller</code> deployment is used as a global switch to control whether to create a load-balancer record. The <code>enableLb</code> parameter added in the subnet is used to control whether the subnet is associated with a load-balancer record. After the previous version is upgraded to v1.12.0, the <code>enableLb</code> parameter of the subnet will automatically inherit the value of the original global switch parameter.</p>"},{"location":"en/guide/setup-options/#networkpolicy-settings","title":"NetworkPolicy Settings","text":"<p>Kube-OVN uses ACLs in OVN to implement NetworkPolicy. Users can choose to disable the NetworkPolicy feature or use the Cilium Chain approach to implement NetworkPolicy using eBPF. In this case, the NetworkPolicy feature of Kube-OVN can be disabled to achieve better performance on the control plane and data plane.</p> <p>This feature can be configured in the installation script:</p> <pre><code>ENABLE_NP=false\n</code></pre> <p>It can also be configured after installation by changing the args of the <code>kube-ovn-controller</code> Deployment:</p> <pre><code>args:\n- --enable-np=false\n</code></pre> <p>NetworkPolicy is enabled by default.</p>"},{"location":"en/guide/setup-options/#eip-and-snat-settings","title":"EIP and SNAT Settings","text":"<p>If the EIP and SNAT capabilities are not required on the default VPC, users can choose to disable them to reduce the performance overhead of <code>kube-ovn-controller</code> in large scale cluster environments and improve processing speed.</p> <p>This feature can be configured in the installation script:</p> <pre><code>ENABLE_EIP_SNAT=false\n</code></pre> <p>It can also be configured after installation by changing the args of the <code>kube-ovn-controller</code> Deployment:</p> <pre><code>args:\n- --enable-eip-snat=false\n</code></pre> <p>EIP and SNAT is enabled by default. More information can refer to EIP and SNAT\u3002</p>"},{"location":"en/guide/setup-options/#centralized-gateway-ecmp-settings","title":"Centralized Gateway ECMP Settings","text":"<p>The centralized gateway supports two mode of high availability, primary-backup and ECMP. If you want to enable ECMP mode, you need to change the args of <code>kube-ovn-controller</code> Deployment:</p> <pre><code>args:\n- --enable-ecmp=true </code></pre> <p>Centralized gateway default installation under the primary-backup mode, more gateway-related content please refer to Config Subnet.</p> <p>The spec field <code>enableEcmp</code> has been added to the subnet crd definition since Kube-OVN v1.12.0 to migrate the ECMP switch to the subnet level. You can set whether to enable ECMP mode based on different subnets. The <code>enable-ecmp</code> parameter in the <code>kube-ovn-controller</code> deployment is no longer used. After the previous version is upgraded to v1.12.0, the subnet switch will automatically inherit the value of the original global switch parameter.</p>"},{"location":"en/guide/setup-options/#kubevirt-vm-fixed-address-settings","title":"Kubevirt VM Fixed Address Settings","text":"<p>For VM instances created by Kubevirt, <code>kube-ovn-controller</code> can assign and manage IP addresses in a similar way to the StatefulSet Pod. This allows VM instances address fixed during start-up, shutdown, upgrade, migration, and other operations throughout their lifecycle, making them more compatible with the actual virtualization user experience.</p> <p>This feature is enabled by default after v1.10.6. To disable this feature, you need to change the following args in the <code>kube-ovn-controller</code> Deployment:</p> <pre><code>args:\n- --keep-vm-ip=false\n</code></pre>"},{"location":"en/guide/setup-options/#cni-settings","title":"CNI Settings","text":"<p>By default, Kube-OVN installs the CNI binary in the <code>/opt/cni/bin</code> directory and the CNI configuration file <code>01-kube-ovn.conflist</code> in the <code>/etc/cni/net.d</code> directory. If you need to change the installation location and the priority of the CNI configuration file, you can modify the following parameters of the installation script.</p> <pre><code>CNI_CONF_DIR=\"/etc/cni/net.d\"\nCNI_BIN_DIR=\"/opt/cni/bin\"\nCNI_CONFIG_PRIORITY=\"01\"\n</code></pre> <p>Or change the Volume mount and args of the <code>kube-ovn-cni</code> DaemonSet after installation:</p> <pre><code>volumes:\n- name: cni-conf\nhostPath:\npath: \"/etc/cni/net.d\"\n- name: cni-bin\nhostPath:\npath:\"/opt/cni/bin\"\n...\nargs:\n- --cni-conf-name=01-kube-ovn.conflist\n</code></pre>"},{"location":"en/guide/setup-options/#tunnel-type-settings","title":"Tunnel Type Settings","text":"<p>The default encapsulation mode of Kube-OVN Overlay is Geneve, if you want to change it to Vxlan or STT, please adjust the following parameters in the installation script:</p> <pre><code>TUNNEL_TYPE=\"vxlan\"\n</code></pre> <p>Or change the environment variables of <code>ovs-ovn</code> DaemonSet after installation:</p> <pre><code>env:\n- name: TUNNEL_TYPE\nvalue: \"vxlan\"\n</code></pre> <p>If you need to use the STT tunnel and need to compile additional kernel modules for ovs, please refer to Performance Tunning\u3002</p> <p>Please refer to Tunneling Protocol Selection for the differences between the different protocols in practice.</p>"},{"location":"en/guide/setup-options/#ssl-settings","title":"SSL Settings","text":"<p>The OVN DB API interface supports SSL encryption to secure the connection. To enable it, adjust the following parameters in the installation script:</p> <pre><code>ENABLE_SSL=true\n</code></pre> <p>The SSL is disabled by default.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/static-ip-mac/","title":"Fixed Addresses","text":"<p>By default, Kube-OVN randomly assigns IPs and Macs based on the Subnet to which the Pod's Namespace belongs. For workloads that require fixed addresses, Kube-OVN provides multiple methods of fixing addresses depending on the scenario.</p> <ul> <li>Single Pod fixed IP/Mac.</li> <li>Workload IP Pool to specify fixed addresses.</li> <li>StatefulSet fixed address.</li> <li>KubeVirt VM fixed address.</li> </ul>"},{"location":"en/guide/static-ip-mac/#single-pod-fixed-ipmac","title":"Single Pod Fixed IP/Mac","text":"<p>You can specify the IP/Mac required for the Pod by annotation when creating the Pod. The <code>kube-ovn-controller</code> will skip the address random assignment phase and use the specified address directly after conflict detection, as follows:</p> <pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nname: ippool\nlabels:\napp: ippool\nspec:\nreplicas: 2\nselector:\nmatchLabels:\napp: ippool\ntemplate:\nmetadata:\nlabels:\napp: ippool\nannotations:\novn.kubernetes.io/ip_pool: 10.16.0.15,10.16.0.16,10.16.0.17 // for dualstack ippool use semicolon to separate addresses 10.16.0.15,fd00:10:16::000E;10.16.0.16,fd00:10:16::0\nspec:\ncontainers:\n- name: ippool\nimage: docker.io/library/nginx:alpine\n</code></pre> <p>The following points need to be noted when using annotation.</p> <ol> <li>The IP/Mac used cannot conflict with an existing IP/Mac.</li> <li>The IP must be in the CIDR range of the Subnet it belongs to.</li> <li>You can specify only IP or Mac. When you specify only one, the other one will be assigned randomly.</li> </ol>"},{"location":"en/guide/static-ip-mac/#workload-ip-pool","title":"Workload IP Pool","text":"<p>Kube-OVN supports setting fixed IPs for Workloads (Deployment/StatefulSet/DaemonSet/Job/CronJob) via annotation <code>ovn.kubernetes.io/ip_pool</code>. <code>kube-ovn-controller</code> will automatically select the IP specified in <code>ovn.kubernetes.io/ip_pool</code> and perform conflict detection.</p> <p>The Annotation of the IP Pool needs to be added to the <code>annotation</code> field in the <code>template</code>. In addition to Kubernetes built-in workload types, other user-defined workloads can also be assigned fixed addresses using the same approach.</p>"},{"location":"en/guide/static-ip-mac/#deployment-with-fixed-ips","title":"Deployment With Fixed IPs","text":"<pre><code>apiVersion: apps/v1\nkind: Deployment\nmetadata:\nnamespace: ls1\nname: starter-backend\nlabels:\napp: starter-backend\nspec:\nreplicas: 2\nselector:\nmatchLabels:\napp: starter-backend\ntemplate:\nmetadata:\nlabels:\napp: starter-backend\nannotations:\novn.kubernetes.io/ip_pool: 10.16.0.15,10.16.0.16,10.16.0.17 // for dualstack ippool use semicolon to separate addresses 10.16.0.15,fd00:10:16::000E;10.16.0.16,fd00:10:16::000F;10.16.0.17,fd00:10:16::0010\nspec:\ncontainers:\n- name: backend\nimage: docker.io/library/nginx:alpine\n</code></pre> <p>Using a fixed IP for Workload requires the following:</p> <ol> <li>The IP in <code>ovn.kubernetes.io/ip_pool</code> should belong to the CIDR of the Subnet.</li> <li>The IP in <code>ovn.kubernetes.io/ip_pool</code> cannot conflict with an IP already in use.</li> <li>When the number of IPs in <code>ovn.kubernetes.io/ip_pool</code> is less than the number of replicas, the extra Pods will not be created. You need to adjust the number of IPs in <code>ovn.kubernetes.io/ip_pool</code> according to the update policy of the workload and the scaling plan.</li> </ol>"},{"location":"en/guide/static-ip-mac/#statefulset-fixed-address","title":"StatefulSet Fixed Address","text":"<p>StatefulSet supports fixed IP by default, and like other Workload, you can use <code>ovn.kubernetes.io/ip_pool</code> to specify the range of IP used by a Pod.</p> <p>Since StatefulSet is mostly used for stateful services, which have higher requirements for fixed addresses, Kube-OVN has made special enhancements:</p> <ol> <li>Pods are assigned IPs in <code>ovn.kubernetes.io/ip_pool</code> in order. For example, if the name of the StatefulSet is web, web-0 will use the first IP in <code>ovn.kubernetes.io/ip_pool</code>, web-1 will use the second IP, and so on.</li> <li>The logical_switch_port in the OVN is not deleted during update or deletion of the StatefulSet Pod, and the newly generated Pod directly reuses the old logical port information. Pods can therefore reuse IP/Mac and other network information to achieve similar state retention as StatefulSet Volumes.</li> <li>Based on the capabilities of 2, for StatefulSet without the <code>ovn.kubernetes.io/ip_pool</code> annotation, a Pod is randomly assigned an IP/Mac when it is first generated, and then the network information remains fixed for the lifetime of the StatefulSet.</li> </ol>"},{"location":"en/guide/static-ip-mac/#statefulset-example","title":"StatefulSet Example","text":"<pre><code>apiVersion: apps/v1\nkind: StatefulSet\nmetadata:\nname: web\nspec:\nserviceName: \"nginx\"\nreplicas: 2\nselector:\nmatchLabels:\napp: nginx\ntemplate:\nmetadata:\nlabels:\napp: nginx\nspec:\ncontainers:\n- name: nginx\nimage: docker.io/library/nginx:alpine\nports:\n- containerPort: 80\nname: web\n</code></pre> <p>You can try to delete the Pod under StatefulSet to observe if the Pod IP changes.</p>"},{"location":"en/guide/static-ip-mac/#kubevirt-vm-fixed-address","title":"KubeVirt VM Fixed Address","text":"<p>For VM instances created by KubeVirt, <code>kube-ovn-controller</code> can assign and manage IP addresses in a similar way to the StatefulSet Pod. This allows VM instances address fixed during start-up, shutdown, upgrade, migration, and other operations throughout their lifecycle, making them more compatible with the actual virtualization user experience.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/subnet/","title":"Config Subnet","text":"<p>Subnet is a core concept and basic unit of use in Kube-OVN, and Kube-OVN organizes IP and network configuration in terms of Subnet. Each Namespace can belong to a specific Subnet, and Pods under the Namespace automatically obtain IPs from the Subnet they belong to and share the network configuration (CIDR, gateway type, access control, NAT control, etc.).</p> <p>Unlike other CNI implementations where each node is bound to a subnet, in Kube-OVN the Subnet is a global level virtual network configuration, and the addresses of one Subnet can be distributed on any node.</p> <p></p> <p>There are some differences in the usage and configuration of Overlay and Underlay Subnets, and this document will describe the common configurations and differentiated features of the different types of Subnets.</p>"},{"location":"en/guide/subnet/#default-subnet","title":"Default Subnet","text":"<p>To make it easier for users to get started quickly, Kube-OVN has a built-in default Subnet, all Namespaces that do not explicitly declare subnet affiliation are automatically assigned IPs from the default subnet and the network information. The configuration of this Subnet is specified at installation time, you can refer to Built-in Network Settings for more details. To change the CIDR of the default Subnet after installation please refer to Change Subnet CIDR.</p> <p>In Overlay mode, the default Subnet uses a distributed gateway and NAT translation for outbound traffic, which behaves much the same as the Flannel's default behavior, allowing users to use most of the network features without additional configuration.</p> <p>In Underlay mode, the default Subnet uses the physical gateway as the outgoing gateway and enables arping to check network connectivity.</p>"},{"location":"en/guide/subnet/#check-the-default-subnet","title":"Check the Default Subnet","text":"<p>The <code>default</code> field in the default Subnet spec is set to <code>true</code>, and there is only one default Subnet in a cluster, named <code>ovn-default</code>.</p> <pre><code># kubectl get subnet ovn-default -o yaml\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n  creationTimestamp: \"2019-08-06T09:33:43Z\"\ngeneration: 1\nname: ovn-default\n  resourceVersion: \"1571334\"\nselfLink: /apis/kubeovn.io/v1/subnets/ovn-default\n  uid: 7e2451f8-fb44-4f7f-b3e0-cfd27f6fd5d6\nspec:\n  cidrBlock: 10.16.0.0/16\n  default: true\nexcludeIps:\n  - 10.16.0.1\n  gateway: 10.16.0.1\n  gatewayType: distributed\n  natOutgoing: true\nprivate: false\nprotocol: IPv4\n</code></pre>"},{"location":"en/guide/subnet/#join-subnet","title":"Join Subnet","text":"<p>In the Kubernetes network specification, it is required that Nodes can communicate directly with all Pods. To achieve this in Overlay network mode, Kube-OVN creates a <code>join</code> Subnet and creates a virtual NIC <code>ovn0</code> at each node that connect to the <code>join</code> subnet, through which the nodes and Pods can communicate with each other.</p> <p>The configuration of this Subnet is specified at installation time, you can refer to Built-in Network Settings for more details. To change the CIDR of the Join Subnet after installation please refer to Change Join CIDR.</p>"},{"location":"en/guide/subnet/#check-the-join-subnet","title":"Check the Join Subnet","text":"<p>The default name of this subnet is <code>join</code>. There is generally no need to make changes to the network configuration except the CIDR.</p> <pre><code># kubectl get subnet join -o yaml\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n  creationTimestamp: \"2019-08-06T09:33:43Z\"\ngeneration: 1\nname: join\n  resourceVersion: \"1571333\"\nselfLink: /apis/kubeovn.io/v1/subnets/join\n  uid: 9c744810-c678-4d50-8a7d-b8ec12ef91b8\nspec:\n  cidrBlock: 100.64.0.0/16\n  default: false\nexcludeIps:\n  - 100.64.0.1\n  gateway: 100.64.0.1\n  gatewayNode: \"\"\ngatewayType: \"\"\nnatOutgoing: false\nprivate: false\nprotocol: IPv4\n</code></pre> <p>Check the ovn0 NIC at the node:</p> <pre><code># ifconfig ovn0\novn0: flags=4163&lt;UP,BROADCAST,RUNNING,MULTICAST&gt;  mtu 1420\ninet 100.64.0.4  netmask 255.255.0.0  broadcast 100.64.255.255\n        inet6 fe80::800:ff:fe40:5  prefixlen 64  scopeid 0x20&lt;link&gt;\n        ether 0a:00:00:40:00:05  txqueuelen 1000  (Ethernet)\nRX packets 18  bytes 1428 (1.3 KiB)\nRX errors 0  dropped 0  overruns 0  frame 0\nTX packets 19  bytes 1810 (1.7 KiB)\nTX errors 0  dropped 0 overruns 0  carrier 0  collisions 0\n</code></pre>"},{"location":"en/guide/subnet/#create-custom-subnets","title":"Create Custom Subnets","text":"<p>Here we describe the basic operation of how to create a Subnet and associate it with a Namespace, for more advanced configuration, please refer to the subsequent content.</p>"},{"location":"en/guide/subnet/#create-subnet","title":"Create Subnet","text":"<pre><code>cat &lt;&lt;EOF | kubectl create -f -\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n  name: subnet1\nspec:\n  protocol: IPv4\n  cidrBlock: 10.66.0.0/16\n  excludeIps:\n  - 10.66.0.1..10.66.0.10\n  - 10.66.0.101..10.66.0.151\n  gateway: 10.66.0.1\n  gatewayType: distributed\n  natOutgoing: true\n  routeTable: \"\"\n  namespaces:\n  - ns1\n  - ns2\nEOF\n</code></pre> <ul> <li><code>cidrBlock</code>: Subnet CIDR range, different Subnet CIDRs under the same VPC cannot overlap.</li> <li><code>excludeIps</code>: The address list is reserved so that the container network will not automatically assign addresses in the list, which can be used as a fixed IP address assignment segment or to avoid conflicts with existing devices in the physical network in Underlay mode.</li> <li><code>gateway</code>\uff1aFor this subnet gateway address, Kube-OVN will automatically assign the corresponding logical gateway in Overlay mode, and the address should be the underlying physical gateway address in Underlay mode.</li> <li><code>namespaces</code>: Bind the list of Namespace for this Subnet. Pods under the Namespace will be assigned addresses from the current Subnet after binding.</li> <li><code>routeTable</code>: Associate the route table, default is main table, route table definition please defer to Static Routes</li> </ul>"},{"location":"en/guide/subnet/#create-pod-in-the-subnet","title":"Create Pod in the Subnet","text":"<pre><code># kubectl create ns ns1\nnamespace/ns1 created\n\n# kubectl run nginx --image=docker.io/library/nginx:alpine -n ns1\ndeployment.apps/nginx created\n\n# kubectl get pod -n ns1 -o wide\nNAME                     READY   STATUS    RESTARTS   AGE   IP           NODE    NOMINATED NODE   READINESS GATES\nnginx-74d5899f46-n8wtg   1/1     Running   0          10s   10.66.0.11   node1   &lt;none&gt;           &lt;none&gt;\n</code></pre>"},{"location":"en/guide/subnet/#overlay-subnet-gateway-settings","title":"Overlay Subnet Gateway Settings","text":"<p>This feature only works for Overlay mode Subnets, Underlay type Subnets need to use the underlying physical gateway to access the external network.</p> <p>Pods under the Overlay Subnet need to access the external network through a gateway, and Kube-OVN currently supports two types of gateways: distributed gateway and centralized gateway which can be changed in the Subnet spec.</p> <p>Both types of gateways support the <code>natOutgoing</code> setting, which allows the user to choose whether snat is required when the Pod accesses the external network.</p>"},{"location":"en/guide/subnet/#distributed-gateway","title":"Distributed Gateway","text":"<p>The default type of gateway for the Subnet, each node will act as a gateway for the pod on the current node to access the external network. The packets from container will flow into the host network stack from the local <code>ovn0</code> NIC, and then forwarding the network according to the host's routing rules. When <code>natOutgoing</code> is <code>true</code>, the Pod will use the IP of the current host when accessing the external network.</p> <p></p> <p>Example of a Subnet, where the <code>gatewayType</code> field is <code>distributed</code>:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: distributed\nspec:\nprotocol: IPv4\ncidrBlock: 10.166.0.0/16\ndefault: false\nexcludeIps:\n- 10.166.0.1\ngateway: 10.166.0.1\ngatewayType: distributed\nnatOutgoing: true\n</code></pre>"},{"location":"en/guide/subnet/#centralized-gateway","title":"Centralized Gateway","text":"<p>If you want traffic within the Subnet to access the external network using a fixed IP for security operations such as auditing and whitelisting, you can set the gateway type in the Subnet to centralized. In centralized gateway mode, packets from Pods accessing the external network are first routed to the <code>ovn0</code> NIC of a specific nodes, and then outbound through the host's routing rules. When <code>natOutgoing</code> is <code>true</code>, the Pod will use the IP of a specific nodes when accessing the external network.</p> <p>The centralized gateway example is as follows, where the <code>gatewayType</code> field is <code>centralized</code> and <code>gatewayNode</code> is the NodeName of the particular machine in Kubernetes.</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: centralized\nspec:\nprotocol: IPv4\ncidrBlock: 10.166.0.0/16\ndefault: false\nexcludeIps:\n- 10.166.0.1\ngateway: 10.166.0.1\ngatewayType: centralized\ngatewayNode: \"node1,node2\"\nnatOutgoing: true\n</code></pre> <ul> <li>If a centralized gateway wants to specify a specific NIC of a machine for outbound networking, <code>gatewayNode</code> format can be changed to <code>kube-ovn-worker:172.18.0.2, kube-ovn-control-plane:172.18.0.3</code>.</li> <li>The centralized gateway defaults to primary-backup mode, with only the primary node performing traffic forwarding.   If you need to switch to ECMP mode, please refer to ECMP Settings.</li> <li>The spec field <code>enableEcmp</code> has been added to the subnet crd definition since Kube-OVN v1.12.0 to migrate the ECMP switch to the subnet level. You can set whether to enable ECMP mode based on different subnets. The <code>enable-ecmp</code> parameter in the <code>kube-ovn-controller</code> deployment is no longer used. After the previous version is upgraded to v1.12.0, the subnet switch will automatically inherit the value of the original global switch parameter.</li> </ul>"},{"location":"en/guide/subnet/#subnet-acl","title":"Subnet ACL","text":"<p>For scenarios with fine-grained ACL control, Subnet of Kube-OVN provides ACL to enable fine-grained rules.</p> <p>The ACL rules in Subnet are the same as the ACL rules in OVN, and you can refer to ovn-nb ACL Table for more details. The supported filed in <code>match</code> can refer to ovn-sb Logical Flow Table.</p> <p>Example of an ACL rule that allows Pods with IP address <code>10.10.0.2</code> to access all addresses, but does not allow other addresses to access itself, is as follows:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: acl\nspec:\nacls:\n- action: drop\ndirection: to-lport\nmatch: ip4.dst == 10.10.0.2 &amp;&amp; ip\npriority: 1002\n- action: allow-related\ndirection: from-lport\nmatch: ip4.src == 10.10.0.2 &amp;&amp; ip\npriority: 1002\ncidrBlock: 10.10.0.0/24\n</code></pre>"},{"location":"en/guide/subnet/#subnet-isolation","title":"Subnet Isolation","text":"<p>The function of Subnet ACL can cover the function of Subnet isolation with better flexibility, we recommend using Subnet ACL to do the corresponding configuration.</p> <p>By default the Subnets created by Kube-OVN can communicate with each other, and Pods can also access external networks through the gateway.</p> <p>To control access between Subnets, set <code>private</code> to true in the subnet spec, and the Subnet will be isolated from other Subnets and external networks and can only communicate within the Subnet. If you want to open a whitelist, you can set it by <code>allowSubnets</code>. The CIDRs in <code>allowSubnets</code> can access the Subnet bidirectionally.</p>"},{"location":"en/guide/subnet/#enable-subnet-isolation-examples","title":"Enable Subnet Isolation Examples","text":"<pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: private\nspec:\nprotocol: IPv4\ndefault: false\nnamespaces:\n- ns1\n- ns2\ncidrBlock: 10.69.0.0/16\nprivate: true\nallowSubnets:\n- 10.16.0.0/16\n- 10.18.0.0/16\n</code></pre>"},{"location":"en/guide/subnet/#underlay-settings","title":"Underlay Settings","text":"<p>This part of the feature is only available for Underlay type Subnets.</p> <ul> <li><code>vlan</code>: If an Underlay network is used, this field is used to control which Vlan CR the Subnet is bound to. This option defaults to the empty string, meaning that the Underlay network is not used.</li> <li><code>logicalGateway</code>: Some Underlay environments are pure Layer 2 networks, with no physical Layer 3 gateway. In this case a virtual gateway can be set up with the OVN to connect the Underlay and Overlay networks. The default value is: <code>false</code>.</li> </ul>"},{"location":"en/guide/subnet/#gateway-check-settings","title":"Gateway Check Settings","text":"<p>By default <code>kube-ovn-cni</code> will request the gateway using ICMP or ARP protocol after starting the Pod and wait for the return to verify that the network is working properly. Some Underlay environment gateways cannot respond to ARP requests, or scenarios that do not require external connectivity, the checking can be disabled .</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: disable-gw-check\nspec:\ndisableGatewayCheck: true\n</code></pre>"},{"location":"en/guide/subnet/#multicast-snoop-setting","title":"Multicast-Snoop Setting","text":"<p>By default, if a Pod in a subnet sends a multicast packet, OVN's default behavior is to broadcast the multicast packet to all Pods in the subnet. If turned on the subnet's multicast snoop switch, OVN will forward based on the multicast table <code>Multicast_Group</code> in the <code>South Database</code> instead of broadcasting.</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: sample1\nspec:\nenableMulticastSnoop: true\n</code></pre>"},{"location":"en/guide/subnet/#other-advanced-settings","title":"Other Advanced Settings","text":"<ul> <li>Configure IPPool</li> <li>Default VPC NAT Policy Rule</li> <li>Manage QoS</li> <li>Manage Multiple Interface</li> <li>DHCP</li> <li>External Gateway</li> <li>Cluster Inter-Connection with OVN-IC</li> <li>VIP Reservation</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/vpc-qos/","title":"VPC QoS","text":"<p>Kube-OVN supports using QoSPolicy CRD to limit the traffic rate of custom VPC.</p>"},{"location":"en/guide/vpc-qos/#eip-qos","title":"EIP QoS","text":"<p>Limit the speed of EIP to 1Mbps and the priority to 1, and <code>shared=false</code> here means that this QoSPolicy can only be used for this EIP and support dynamically modifying QoSPolicy to change QoS rules.</p> <p>The QoSPolicy configuration is as follows:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: QoSPolicy\nmetadata:\nname: qos-eip-example\nspec:\nshared: false\nbindingType: EIP\nbandwidthLimitRules:\n- name: eip-ingress\nrateMax: \"1\" # Mbps\nburstMax: \"1\" # Mbps\npriority: 1\ndirection: ingress\n- name: eip-egress\nrateMax: \"1\" # Mbps\nburstMax: \"1\" # Mbps\npriority: 1\ndirection: egress\n</code></pre> <p>The IptablesEIP configuration is as follows:</p> <pre><code>kind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-1\nspec:\nnatGwDp: gw1\nqosPolicy: qos-eip-example\n</code></pre> <p>The value of <code>.spec.qosPolicy</code> supports being specified during creation and also supports modification after creation.</p>"},{"location":"en/guide/vpc-qos/#view-eips-with-qos-enabled","title":"View EIPs with QoS enabled","text":"<p>View the corresponding EIPs that have been set up using <code>label</code>:</p> <pre><code># kubectl get eip  -l ovn.kubernetes.io/qos=qos-eip-example\nNAME    IP             MAC                 NAT   NATGWDP   READY\neip-1   172.18.11.24   00:00:00:34:41:0B   fip   gw1       true\n</code></pre>"},{"location":"en/guide/vpc-qos/#qos-for-vpc-natgw-net1-nic","title":"QoS for VPC NATGW net1 NIC","text":"<p>Limit the speed of the net1 NIC on VPC NATGW to 10Mbps and set the priority to 3. Here <code>shared=true</code>, which means that this QoSPolicy can be used by multiple resources at the same time, and does not allow the modification of the contents of the QoSPolicy in this scenario.</p> <p>The QoSPolicy configuration is as follows:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: QoSPolicy\nmetadata:\nname: qos-natgw-example\nspec:\nshared: true\nbindingType: NATGW\nbandwidthLimitRules:\n- name: net1-ingress\ninterface: net1\nrateMax: \"10\" # Mbps\nburstMax: \"10\" # Mbps\npriority: 3\ndirection: ingress\n- name: net1-egress\ninterface: net1\nrateMax: \"10\" # Mbps\nburstMax: \"10\" # Mbps\npriority: 3\ndirection: egress\n</code></pre> <p>The VpcNatGateway configuration is as follows:</p> <pre><code>kind: VpcNatGateway\napiVersion: kubeovn.io/v1\nmetadata:\nname: gw1\nspec:\nvpc: test-vpc-1\nsubnet: net1\nlanIp: 10.0.1.254\nqosPolicy: qos-natgw-example\nselector:\n- \"kubernetes.io/hostname: kube-ovn-worker\"\n- \"kubernetes.io/os: linux\"\n</code></pre> <p>The value of <code>.spec.qosPolicy</code> supports both creation and subsequent modification.</p>"},{"location":"en/guide/vpc-qos/#qos-for-specific-traffic-on-net1-nic","title":"QoS for specific traffic on net1 NIC","text":"<p>Limit the specific traffic on net1 NIC to 5Mbps and set the priority to 2. Here <code>shared=true</code>, which means that this QoSPolicy can be used by multiple resources at the same time, and does not allow the modification of the contents of the QoSPolicy in this scenario.</p> <p>The QoSPolicy configuration is as follows:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: QoSPolicy\nmetadata:\nname: qos-natgw-example\nspec:\nshared: true\nbindingType: NATGW\nbandwidthLimitRules:\n- name: net1-extip-ingress\ninterface: net1\nrateMax: \"5\" # Mbps\nburstMax: \"5\" # Mbps\npriority: 2\ndirection: ingress\nmatchType: ip\nmatchValue: src 172.18.11.22/32\n- name: net1-extip-egress\ninterface: net1\nrateMax: \"5\" # Mbps\nburstMax: \"5\" # Mbps\npriority: 2\ndirection: egress\nmatchType: ip\nmatchValue: dst 172.18.11.23/32\n</code></pre> <p>The VpcNatGateway configuration is as follows:</p> <pre><code>kind: VpcNatGateway\napiVersion: kubeovn.io/v1\nmetadata:\nname: gw1\nspec:\nvpc: test-vpc-1\nsubnet: net1\nlanIp: 10.0.1.254\nqosPolicy: qos-natgw-example\nselector:\n- \"kubernetes.io/hostname: kube-ovn-worker\"\n- \"kubernetes.io/os: linux\"\n</code></pre>"},{"location":"en/guide/vpc-qos/#view-natgws-with-qos-enabled","title":"View NATGWs with QoS enabled","text":"<p>View the corresponding NATGWs that have been set up using <code>label</code>:</p> <pre><code># kubectl get vpc-nat-gw  -l ovn.kubernetes.io/qos=qos-natgw-example\nNAME   VPC          SUBNET   LANIP\ngw1    test-vpc-1   net1     10.0.1.254\n</code></pre>"},{"location":"en/guide/vpc-qos/#view-qos-rules","title":"View QoS rules","text":"<pre><code># kubectl get qos -A\nNAME                SHARED   BINDINGTYPE\nqos-eip-example     false    EIP\nqos-natgw-example   true     NATGW\n</code></pre>"},{"location":"en/guide/vpc-qos/#limitations","title":"Limitations","text":"<ul> <li>QoSPolicy can only be deleted when it is not in use. Therefore, before deleting the QoSPolicy, please check the EIP and NATGW that have enabled QoS, and remove their <code>spec.qosPolicy</code> configuration.</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/vpc/","title":"Config VPC","text":"<p>Kube-OVN supports multi-tenant isolation level VPC networks. Different VPC networks are independent of each other and can be configured separately with Subnet CIDRs, routing policies, security policies, outbound gateways, EIP, etc.</p> <p>VPC is mainly used in scenarios where there requires strong isolation of multi-tenant networks and some Kubernetes networking features conflict under multi-tenant networks. For example, node and pod access, NodePort functionality, network access-based health checks, and DNS capabilities are not supported in multi-tenant network scenarios at this time. In order to facilitate common Kubernetes usage scenarios, Kube-OVN has a special design for the default VPC where the Subnet under the VPC can meet the Kubernetes specification. The custom VPC supports static routing, EIP and NAT gateways as described in this document. Common isolation requirements can be achieved through network policies and Subnet ACLs under the default VPC, so before using a custom VPC, please make sure whether you need VPC-level isolation and understand the limitations under the custom VPC. For Underlay subnets, physical switches are responsible for data-plane forwarding, so VPCs cannot isolate Underlay subnets.</p> <p></p>"},{"location":"en/guide/vpc/#creating-custom-vpcs","title":"Creating Custom VPCs","text":"<p>Create two VPCs:</p> <pre><code>kind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: test-vpc-1\nspec:\nnamespaces:\n- ns1\n---\nkind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: test-vpc-2\nspec:\nnamespaces:\n- ns2\n</code></pre> <ul> <li><code>namespaces</code>: Limit which namespaces can use this VPC. If empty, all namespaces can use this VPC.</li> </ul> <p>Create two Subnets, belonging to two different VPCs and having the same CIDR:</p> <pre><code>kind: Subnet\napiVersion: kubeovn.io/v1\nmetadata:\nname: net1\nspec:\nvpc: test-vpc-1\ncidrBlock: 10.0.1.0/24\nprotocol: IPv4\nnamespaces:\n- ns1\n---\nkind: Subnet\napiVersion: kubeovn.io/v1\nmetadata:\nname: net2\nspec:\nvpc: test-vpc-2\ncidrBlock: 10.0.1.0/24\nprotocol: IPv4\nnamespaces:\n- ns2\n</code></pre> <p>Create Pods under two separate Namespaces:</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nannotations:\novn.kubernetes.io/logical_switch: net1\nnamespace: ns1\nname: vpc1-pod\nspec:\ncontainers:\n- name: vpc1-pod\nimage: docker.io/library/nginx:alpine\n---\napiVersion: v1\nkind: Pod\nmetadata:\nannotations:\novn.kubernetes.io/logical_switch: net2\nnamespace: ns2\nname: vpc2-pod\nspec:\ncontainers:\n- name: vpc2-pod\nimage: docker.io/library/nginx:alpine\n</code></pre> <p>After running successfully, you can observe that the two Pod addresses belong to the same CIDR, but the two Pods cannot access each other because they are running on different tenant VPCs.</p>"},{"location":"en/guide/vpc/#custom-vpc-pod-supports-livenessprobe-and-readinessprobe","title":"Custom VPC Pod supports livenessProbe and readinessProbe","text":"<p>Since the Pods under the custom VPC do not communicate with the network of the node, the probe packets sent by the kubelet cannot reach the Pods in the custom VPC. Kube-OVN uses TProxy to redirect the detection packets sent by kubelet to Pods in the custom VPC to achieve this function.</p> <p>The configuration method is as follows, add the parameter <code>--enable-tproxy=true</code> in Daemonset <code>kube-ovn-cni</code>:</p> <pre><code>spec:\ntemplate:\nspec:\ncontainers:\n- args:\n- --enable-tproxy=true\n</code></pre> <p>Restrictions for this feature:</p> <ol> <li>When Pods under different VPCs have the same IP under the same node, the detection function fails.</li> <li>Currently, only <code>tcpSocket</code> and <code>httpGet</code> are supported.</li> </ol>"},{"location":"en/guide/vpc/#create-vpc-nat-gateway","title":"Create VPC NAT Gateway","text":"<p>Subnets under custom VPCs do not support distributed gateways and centralized gateways under default VPCs.</p> <p>Pod access to the external network within the VPC requires a VPC gateway, which bridges the physical and tenant networks and provides floating IP, SNAT and DNAT capabilities.</p> <p>The VPC gateway function relies on Multus-CNI function, please refer to multus-cni.</p>"},{"location":"en/guide/vpc/#configuring-the-external-network","title":"Configuring the External Network","text":"<pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: ovn-vpc-external-network\nspec:\nprotocol: IPv4\nprovider: ovn-vpc-external-network.kube-system\ncidrBlock: 192.168.0.0/24\ngateway: 192.168.0.1  # IP address of the physical gateway\nexcludeIps:\n- 192.168.0.1..192.168.0.10\n---\napiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: ovn-vpc-external-network\nnamespace: kube-system\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.0\",\n\"type\": \"macvlan\",\n\"master\": \"eth1\",\n\"mode\": \"bridge\",\n\"ipam\": {\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"ovn-vpc-external-network.kube-system\"\n}\n}'\n</code></pre> <ul> <li>This Subnet is used to manage the available external addresses and the address will be allocated to VPC NAT Gateway through Macvlan, so please communicate with your network management to give you the available physical segment IPs.</li> <li>The VPC gateway uses Macvlan for physical network configuration, and <code>master</code> of <code>NetworkAttachmentDefinition</code> should be the NIC name of the corresponding physical network NIC.</li> <li><code>name</code>: External network name.</li> </ul> <p>For macvlan mode, the nic will send packets directly through that node NIC, relying on the underlying network devices for L2/L3 level forwarding capabilities. You need to configure the corresponding gateway, Vlan and security policy in the underlying network device in advance.</p> <ol> <li>For OpenStack VM environments, you need to turn off <code>PortSecurity</code> on the corresponding network port.</li> <li>For VMware vSwitch networks, <code>MAC Address Changes</code>, <code>Forged Transmits</code> and <code>Promiscuous Mode Operation</code> should be set to <code>allow</code>.</li> <li>For Hyper-V virtualization,  <code>MAC Address Spoofing</code> should be enabled in VM nic advanced features.</li> <li>Public clouds, such as AWS, GCE, AliCloud, etc., do not support user-defined Mac, so they cannot support Macvlan mode network.</li> <li>Due to the limitations of Macvlan, the Macvlan sub-interface cannot access the parent interface address.</li> <li>If the physical network card corresponds to a switch interface in Trunk mode, a sub-interface needs to be created on the network card and provided to Macvlan for use.</li> </ol>"},{"location":"en/guide/vpc/#enabling-the-vpc-gateway","title":"Enabling the VPC Gateway","text":"<p>VPC gateway functionality needs to be enabled via <code>ovn-vpc-nat-gw-config</code> under <code>kube-system</code>:</p> <pre><code>---\nkind: ConfigMap\napiVersion: v1\nmetadata:\nname: ovn-vpc-nat-config\nnamespace: kube-system\ndata:\nimage: docker.io/kubeovn/vpc-nat-gateway:v1.13.0\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\nname: ovn-vpc-nat-gw-config\nnamespace: kube-system\ndata:\nenable-vpc-nat-gw: 'true'\n</code></pre> <ul> <li><code>image</code>: The image used by the Gateway Pod.</li> <li><code>enable-vpc-nat-gw</code>: Controls whether the VPC Gateway feature is enabled.</li> </ul>"},{"location":"en/guide/vpc/#create-vpc-gateway-and-set-the-default-route","title":"Create VPC Gateway and Set the Default Route","text":"<pre><code>kind: VpcNatGateway\napiVersion: kubeovn.io/v1\nmetadata:\nname: gw1\nspec:\nvpc: test-vpc-1\nsubnet: net1\nlanIp: 10.0.1.254\nselector:\n- \"kubernetes.io/hostname: kube-ovn-worker\"\n- \"kubernetes.io/os: linux\"\nexternalSubnets:\n- ovn-vpc-external-network\n</code></pre> <ul> <li><code>vpc</code>: The VPC to which this VpcNatGateway belongs.</li> <li><code>subnet</code>: A Subnet within the VPC, the VPC Gateway Pod will use <code>lanIp</code> to connect to the tenant network under that subnet.</li> <li><code>lanIp</code>: An unused IP within the <code>subnet</code> that the VPC Gateway Pod will eventually use for the Pod. When configuring routing for a VPC, the  <code>nextHopIP</code> needs to be set to the <code>lanIp</code> of the current VpcNatGateway.</li> <li><code>selector</code>: The node selector for VpcNatGateway Pod has the same format as NodeSelector in Kubernetes.</li> <li><code>externalSubnets</code>: External network used by the VPC gateway, if not configured, <code>ovn-vpc-external-network</code> is used by default, and only one external network is supported in the current version.</li> </ul> <p>Other configurable parameters:</p> <ul> <li><code>tolerations</code>: Configure tolerance for the VPC gateway. For details, see Taints and Tolerations</li> <li><code>affinity</code>: Configure affinity for the Pod or node of the VPC gateway. For details, see Assigning Pods to Nodes</li> </ul>"},{"location":"en/guide/vpc/#create-eip","title":"Create EIP","text":"<p>EIP allows for floating IP, SNAT, and DNAT operations after assigning an IP from an external network segment to a VPC gateway.</p> <p>Randomly assign an address to the EIP:</p> <pre><code>kind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-random\nspec:\nnatGwDp: gw1\n</code></pre> <p>Fixed EIP address assignment:</p> <pre><code>kind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-static\nspec:\nnatGwDp: gw1\nv4ip: 10.0.1.111\n</code></pre> <p>Specify the external network on which the EIP is located:</p> <pre><code>kind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eip-random\nspec:\nnatGwDp: gw1\nexternalSubnet: ovn-vpc-external-network\n</code></pre> <ul> <li><code>externalSubnet</code>: The name of the external network on which the EIP is located. If not specified, it defaults to <code>ovn-vpc-external-network</code>. If specified, it must be one of the <code>externalSubnets</code> of the VPC gateway.</li> </ul>"},{"location":"en/guide/vpc/#create-dnat-rules","title":"Create DNAT Rules","text":"<p>Through the DNAT rules, external can access to an IP and port within a VPC through an EIP and port.</p> <pre><code>kind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eipd01\nspec:\nnatGwDp: gw1\n\n---\nkind: IptablesDnatRule\napiVersion: kubeovn.io/v1\nmetadata:\nname: dnat01\nspec:\neip: eipd01 externalPort: '8888'\ninternalIp: 10.0.1.10\ninternalPort: '80'\nprotocol: tcp\n</code></pre>"},{"location":"en/guide/vpc/#create-snat-rules","title":"Create SNAT Rules","text":"<p>Through SNAT rules, when a Pod in the VPC accesses an external address, it will go through the corresponding EIP for SNAT.</p> <pre><code>---\nkind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eips01\nspec:\nnatGwDp: gw1\n---\nkind: IptablesSnatRule\napiVersion: kubeovn.io/v1\nmetadata:\nname: snat01\nspec:\neip: eips01\ninternalCIDR: 10.0.1.0/24\n</code></pre>"},{"location":"en/guide/vpc/#create-floating-ip","title":"Create Floating IP","text":"<p>Through floating IP rules, one IP in the VPC will be completely mapped to the EIP, and the external can access the IP in the VPC through this EIP. When the IP in the VPC accesses the external address, it will be SNAT to this EIP</p> <pre><code>---\nkind: IptablesEIP\napiVersion: kubeovn.io/v1\nmetadata:\nname: eipf01\nspec:\nnatGwDp: gw1\n\n---\nkind: IptablesFIPRule\napiVersion: kubeovn.io/v1\nmetadata:\nname: fip01\nspec:\neip: eipf01\ninternalIp: 10.0.1.5\n</code></pre>"},{"location":"en/guide/vpc/#custom-routing","title":"Custom Routing","text":"<p>Within the custom VPC, users can customize the routing rules within the VPC and combine it with the gateway for more flexible forwarding. Kube-OVN supports static routes and more flexible policy routes.</p>"},{"location":"en/guide/vpc/#static-routes","title":"Static Routes","text":"<pre><code>kind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: test-vpc-1\nspec:\nstaticRoutes:\n- cidr: 0.0.0.0/0\nnextHopIP: 10.0.1.254\npolicy: policyDst\n- cidr: 172.31.0.0/24\nnextHopIP: 10.0.1.253\npolicy: policySrc\nrouteTable: \"rtb1\"\n</code></pre> <ul> <li><code>policy</code>: Supports destination routing <code>policyDst</code> and source routing <code>policySrc</code>.</li> <li>When there are overlapping routing rules, the rule with the longer CIDR mask has higher priority,   and if the mask length is the same, the destination route has a higher priority over the source route.</li> <li><code>routeTable</code>: You can store the route in specific table, default is main table. Associate with subnet please defer to Create Custom Subnets</li> </ul>"},{"location":"en/guide/vpc/#policy-routes","title":"Policy Routes","text":"<p>Traffic matched by static routes can be controlled at a finer granularity by policy routing. Policy routing provides more precise matching rules, priority control and more forwarding actions. This feature brings the OVN internal logical router policy function directly to the outside world, for more information on its use, please refer to Logical Router Policy.</p> <p>An example of policy routes:</p> <pre><code>kind: Vpc\napiVersion: kubeovn.io/v1\nmetadata:\nname: test-vpc-1\nspec:\npolicyRoutes:\n- action: drop\nmatch: ip4.src==10.0.1.0/24 &amp;&amp; ip4.dst==10.0.1.250\npriority: 11\n- action: reroute\nmatch: ip4.src==10.0.1.0/24\nnextHopIP: 10.0.1.252\npriority: 10\n</code></pre>"},{"location":"en/guide/vpc/#custom-vpc-dns","title":"Custom vpc-dns","text":"<p>Due to the isolation between custom VPCs and default VPC networks, Pods in VPCs cannot use the default coredns service for domain name resolution. If you want to use coredns to resolve Service domain names within the custom VPC, you can use the <code>vpc-dns</code> resource provided by Kube-OVN.</p>"},{"location":"en/guide/vpc/#create-an-additional-network","title":"Create an Additional Network","text":"<pre><code>apiVersion: \"k8s.cni.cncf.io/v1\"\nkind: NetworkAttachmentDefinition\nmetadata:\nname: ovn-nad\nnamespace: default\nspec:\nconfig: '{\n\"cniVersion\": \"0.3.0\",\n\"type\": \"kube-ovn\",\n\"server_socket\": \"/run/openvswitch/kube-ovn-daemon.sock\",\n\"provider\": \"ovn-nad.default.ovn\"\n}'\n</code></pre>"},{"location":"en/guide/vpc/#modify-the-provider-of-the-ovn-default-logical-switch","title":"Modify the Provider of the ovn-default Logical Switch","text":"<p>Modify the provider of ovn-default to the provider <code>ovn-nad.default.ovn</code> configured above in nad\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: ovn-default\nspec:\ncidrBlock: 10.16.0.0/16\ndefault: true\ndisableGatewayCheck: false\ndisableInterConnection: false\nenableDHCP: false\nenableIPv6RA: false\nexcludeIps:\n- 10.16.0.1\ngateway: 10.16.0.1\ngatewayType: distributed\nlogicalGateway: false\nnatOutgoing: true\nprivate: false\nprotocol: IPv4\nprovider: ovn-nad.default.ovn\nvpc: ovn-cluster\n</code></pre>"},{"location":"en/guide/vpc/#modify-the-vpc-dns-configmap","title":"Modify the vpc-dns ConfigMap","text":"<p>Create a ConfigMap in the kube-system namespace, configure the vpc-dns parameters to be used for the subsequent vpc-dns feature activation:</p> <pre><code>apiVersion: v1\nkind: ConfigMap\nmetadata:\nname: vpc-dns-config\nnamespace: kube-system\ndata:\ncoredns-vip: 10.96.0.3\nenable-vpc-dns: true\nnad-name: ovn-nad\nnad-provider: ovn-nad.default.ovn\n</code></pre> <ul> <li><code>enable-vpc-dns</code>: (optional) <code>true</code> to enable the feature, <code>false</code> to disable the feature. Default <code>true</code>.</li> <li><code>coredns-image</code>: (optional): DNS deployment image. Default is the cluster coredns deployment version.</li> <li><code>coredns-template</code>: (optional): URL of the DNS deployment template. Default: <code>yamls/coredns-template.yaml</code> in the current version repository.</li> <li><code>coredns-vip</code>: VIP providing LB service for coredns.</li> <li><code>nad-name</code>: Name of the configured <code>network-attachment-definitions</code> resource.</li> <li><code>nad-provider</code>: Name of the used provider.</li> <li><code>k8s-service-host</code>: (optional) IP used by coredns to access the k8s apiserver service.</li> <li><code>k8s-service-port</code>: (optional) Port used by coredns to access the k8s apiserver service.</li> </ul>"},{"location":"en/guide/vpc/#deploying-vpc-dns-dependent-resources","title":"Deploying VPC-DNS Dependent Resources","text":"<pre><code>apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole\nmetadata:\nlabels:\nkubernetes.io/bootstrapping: rbac-defaults\nname: system:vpc-dns\nrules:\n- apiGroups:\n- \"\"\nresources:\n- endpoints\n- services\n- pods\n- namespaces\nverbs:\n- list\n- watch\n- apiGroups:\n- discovery.k8s.io\nresources:\n- endpointslices\nverbs:\n- list\n- watch\n---\napiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding\nmetadata:\nannotations:\nrbac.authorization.kubernetes.io/autoupdate: \"true\"\nlabels:\nkubernetes.io/bootstrapping: rbac-defaults\nname: vpc-dns\nroleRef:\napiGroup: rbac.authorization.k8s.io\nkind: ClusterRole\nname: system:vpc-dns\nsubjects:\n- kind: ServiceAccount\nname: vpc-dns\nnamespace: kube-system\n---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\nname: vpc-dns\nnamespace: kube-system\n---\napiVersion: v1\nkind: ConfigMap\nmetadata:\nname: vpc-dns-corefile\nnamespace: kube-system\ndata:\nCorefile: |\n.:53 {\nerrors\nhealth {\nlameduck 5s\n}\nready\nkubernetes cluster.local in-addr.arpa ip6.arpa {\npods insecure\nfallthrough in-addr.arpa ip6.arpa\n}\nprometheus :9153\nforward . /etc/resolv.conf {\nprefer_udp\n}\ncache 30\nloop\nreload\nloadbalance\n}\n</code></pre>"},{"location":"en/guide/vpc/#deploy-vpc-dns","title":"Deploy vpc-dns","text":"<pre><code>kind: VpcDns\napiVersion: kubeovn.io/v1\nmetadata:\nname: test-cjh1\nspec:\nvpc: cjh-vpc-1\nsubnet: cjh-subnet-1\n</code></pre> <ul> <li><code>vpc</code>: The VPC name used to deploy the DNS component.</li> <li><code>subnet</code>: The subnet name used to deploy the DNS component.</li> </ul> <p>View resource information:</p> <pre><code>[root@hci-dev-mst-1 kubeovn]# kubectl get vpc-dns\nNAME        ACTIVE   VPC         SUBNET   \ntest-cjh1   false    cjh-vpc-1   cjh-subnet-1   \ntest-cjh2   true     cjh-vpc-1   cjh-subnet-2 </code></pre> <ul> <li><code>ACTIVE</code>: if the custom vpc-dns is ready.</li> </ul>"},{"location":"en/guide/vpc/#restrictions","title":"Restrictions","text":"<ul> <li>Only one custom DNS component will be deployed in one VPC;</li> <li>When multiple VPC-DNS resources (i.e. different subnets in the same VPC) are configured in one VPC, only one VPC-DNS resource with status <code>true</code> will be active, while the others will be <code>false</code>;</li> <li>When the <code>true</code> VPC-DNS is deleted, another <code>false</code> VPC-DNS will be deployed.</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/guide/webhook/","title":"Webhook","text":"<p>Using Webhook, you can verify CRD resources within Kube-OVN. Currently, Webhook mainly performs fixed IP address conflict detection and Subnet CIDR conflict detection, and prompts errors when such conflicts happen.</p> <p>Since Webhook intercepts all Subnet and Pod creation requests, you need to deploy Kube-OVN first and Webhook later.</p>"},{"location":"en/guide/webhook/#install-cert-manager","title":"Install Cert-Manager","text":"<p>Webhook deployment requires certificate, we use cert-manager to generate the associated certificate, we need to deploy cert-manager before deploying Webhook.</p> <p>You can use the following command to deploy cert-manager:</p> <pre><code>kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.8.0/cert-manager.yaml\n</code></pre> <p>More cert-manager usage please refer to cert-manager document\u3002</p>"},{"location":"en/guide/webhook/#install-webhook","title":"Install Webhook","text":"<p>Download Webhook yaml and install:</p> <pre><code># kubectl apply -f https://raw.githubusercontent.com/kubeovn/kube-ovn/master/yamls/webhook.yaml\ndeployment.apps/kube-ovn-webhook created\nservice/kube-ovn-webhook created\nvalidatingwebhookconfiguration.admissionregistration.k8s.io/kube-ovn-webhook created\ncertificate.cert-manager.io/kube-ovn-webhook-serving-cert created\nissuer.cert-manager.io/kube-ovn-webhook-selfsigned-issuer created\n</code></pre>"},{"location":"en/guide/webhook/#verify-webhook-take-effect","title":"Verify Webhook Take Effect","text":"<p>Check the running Pod and get the Pod IP <code>10.16.0.15</code>:</p> <pre><code># kubectl get pod -o wide\nNAME                      READY   STATUS    RESTARTS   AGE     IP           NODE              NOMINATED NODE   READINESS GATES\nstatic-7584848b74-fw9dm   1/1     Running   0          2d13h   10.16.0.15   kube-ovn-worker   &lt;none&gt; </code></pre> <p>Write yaml to create a Pod with the same IP:</p> <pre><code>apiVersion: v1\nkind: Pod\nmetadata:\nannotations:\novn.kubernetes.io/ip_address: 10.16.0.15\novn.kubernetes.io/mac_address: 00:00:00:53:6B:B6\nlabels:\napp: static\nmanagedFields:\nname: staticip-pod\nnamespace: default\nspec:\ncontainers:\n- image: docker.io/library/nginx:alpine\nimagePullPolicy: IfNotPresent\nname: qatest\n</code></pre> <p>When using the above yaml to create a fixed address Pod, it prompts an IP address conflict:</p> <pre><code># kubectl apply -f pod-static.yaml\nError from server (annotation ip address 10.16.0.15 is conflict with ip crd static-7584848b74-fw9dm.default 10.16.0.15): error when creating \"pod-static.yaml\": admission webhook \"pod-ip-validaing.kube-ovn.io\" denied the request: annotation ip address 10.16.0.15 is conflict with ip crd static-7584848b74-fw9dm.default 10.16.0.15\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/ops/change-default-subnet/","title":"Change Subnet CIDR","text":"<p>If a subnet CIDR is created that conflicts or does not meet expectations, it can be modified by following the steps in this document.</p> <p>After modifying the subnet CIDR, the previously created Pods will not be able to access the network properly and need to be rebuilt. Careful consideration is recommended before operating\u3002This document is only for business subnet CIDR changes, if you need to Change the Join subnet CIDR, please refer to Change Join CIDR.</p>"},{"location":"en/ops/change-default-subnet/#edit-subnet","title":"Edit Subnet","text":"<p>Use <code>kubectl edit</code> to modify <code>cidrBlock</code>\uff0c<code>gateway</code> and <code>excludeIps</code>.</p> <pre><code>kubectl edit subnet test-subnet\n</code></pre>"},{"location":"en/ops/change-default-subnet/#rebuild-all-pods-under-this-subnet","title":"Rebuild all Pods under this Subnet","text":"<p>Take the subnet binding <code>test</code> Namespace as example:</p> <pre><code>for pod in $(kubectl get pod --no-headers -n \"$ns\" --field-selector spec.restartPolicy=Always -o custom-columns=NAME:.metadata.name,HOST:spec.hostNetwork | awk '{if ($2!=\"true\") print $1}'); do\nkubectl delete pod \"$pod\" -n test --ignore-not-found\ndone\n</code></pre> <p>If only the default subnet is used, you can delete all Pods that are not in host network mode using the following command:</p> <pre><code>for ns in $(kubectl get ns --no-headers -o custom-columns=NAME:.metadata.name); do\nfor pod in $(kubectl get pod --no-headers -n \"$ns\" --field-selector spec.restartPolicy=Always -o custom-columns=NAME:.metadata.name,HOST:spec.hostNetwork | awk '{if ($2!=\"true\") print $1}'); do\nkubectl delete pod \"$pod\" -n \"$ns\" --ignore-not-found\n  done\ndone\n</code></pre>"},{"location":"en/ops/change-default-subnet/#change-default-subnet-settings","title":"Change Default Subnet Settings","text":"<p>If you are modifying the CIDR for the default Subnet, you also need to change the args of the <code>kube-ovn-controller</code> Deployment:</p> <pre><code>args:\n- --default-cidr=10.17.0.0/16\n- --default-gateway=10.17.0.1\n- --default-exclude-ips=10.17.0.1\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/ops/change-join-subnet/","title":"Change Join Subnet CIDR","text":"<p>If the Join subnet CIDR created conflicts or does not meet expectations, you can use this document to modify.</p> <p>After modifying the Join Subnet CIDR, the previously created Pods will not be able to access the external network normally and need to wait for the rebuild completed.</p>"},{"location":"en/ops/change-join-subnet/#delete-join-subnet","title":"Delete Join Subnet","text":"<pre><code>kubectl patch subnet join --type='json' -p '[{\"op\": \"replace\", \"path\": \"/metadata/finalizers\", \"value\": []}]'\nkubectl delete subnet join\n</code></pre>"},{"location":"en/ops/change-join-subnet/#cleanup-allocated-config","title":"Cleanup Allocated Config","text":"<pre><code>kubectl annotate node ovn.kubernetes.io/allocated=false --all --overwrite\n</code></pre>"},{"location":"en/ops/change-join-subnet/#modify-join-subnet","title":"Modify Join Subnet","text":"<p>Change Join Subnet args in <code>kube-ovn-controller</code>:</p> <pre><code>kubectl edit deployment -n kube-system kube-ovn-controller\n</code></pre> <p>Change the CIDR below:</p> <pre><code>args:\n- --node-switch-cidr=100.51.0.0/16\n</code></pre> <p>Reboot the <code>kube-ovn-controller</code> and rebuild <code>join</code> Subnet:</p> <pre><code>kubectl delete pod -n kube-system -lapp=kube-ovn-controller\n</code></pre> <p>Check the new Join Subnet information:</p> <pre><code># kubectl get subnet\nNAME          PROVIDER   VPC           PROTOCOL   CIDR            PRIVATE   NAT     DEFAULT   GATEWAYTYPE   V4USED   V4AVAILABLE   V6USED   V6AVAILABLE   EXCLUDEIPS\njoin          ovn        ovn-cluster   IPv4       100.51.0.0/16   false     false   false     distributed   2        65531         0        0             [\"100.51.0.1\"]\novn-default   ovn        ovn-cluster   IPv4       10.17.0.0/16    false     true    true      distributed   5        65528         0        0             [\"10.17.0.1\"]\n</code></pre>"},{"location":"en/ops/change-join-subnet/#reconfigure-ovn0-nic-address","title":"Reconfigure ovn0 NIC Address","text":"<p>The <code>ovn0</code> NIC information for each node needs to be re-updated, which can be done by restarting <code>kube-ovn-cni</code>:</p> <pre><code>kubectl delete pod -n kube-system -l app=kube-ovn-cni\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/ops/change-log-level/","title":"Change Log Level","text":"<p>Open <code>kube-ovn.yaml</code> and set the log level in the parameter list of the service startup script, such as:</p> <pre><code>vi kube-ovn.yaml\n# ...\n- name: kube-ovn-controller\n          image: \"docker.io/kubeovn/kube-ovn:v1.13.0\"\nimagePullPolicy: IfNotPresent\n          args:\n          - /kube-ovn/start-controller.sh\n          - --v=3\n# ...\n# The higher the log level, the more detailed the log\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/ops/change-ovn-central-node/","title":"Replace ovn-central Node","text":"<p>Since <code>ovn-nb</code> and <code>ovn-sb</code> within <code>ovn-central</code> create separate etcd-like raft clusters, replacing the <code>ovn-central</code> node requires additional operations to ensure correct cluster state and consistent data. It is recommended that only one node be up and down at a time to avoid the cluster going into an unavailable state and affecting the overall cluster network.</p>"},{"location":"en/ops/change-ovn-central-node/#ovn-central-nodes-offline","title":"ovn-central Nodes Offline","text":"<p>This document use the cluster below to describes how to remove the <code>kube-ovn-control-plane2</code> node from the <code>ovn-central</code> as an example.</p> <pre><code># kubectl -n kube-system get pod -o wide | grep central\novn-central-6bf58cbc97-2cdhg                      1/1     Running   0             21m   172.18.0.3   kube-ovn-control-plane    &lt;none&gt;           &lt;none&gt;\novn-central-6bf58cbc97-crmfp                      1/1     Running   0             21m   172.18.0.5   kube-ovn-control-plane2   &lt;none&gt;           &lt;none&gt;\novn-central-6bf58cbc97-lxmpl                      1/1     Running   0             21m   172.18.0.4   kube-ovn-control-plane3   &lt;none&gt;           &lt;none&gt;\n</code></pre>"},{"location":"en/ops/change-ovn-central-node/#kick-node-in-ovn-nb","title":"Kick Node in ovn-nb","text":"<p>First check the ID of the node within the cluster for subsequent operations.</p> <pre><code># kubectl ko nb status\n1b9a\nName: OVN_Northbound\nCluster ID: 32ca (32ca07fb-739b-4257-b510-12fa18e7cce8)\nServer ID: 1b9a (1b9a5d76-e69b-410c-8085-39943d0cd38c)\nAddress: tcp:[172.18.0.3]:6643\nStatus: cluster member\nRole: leader\nTerm: 1\nLeader: self\nVote: self\n\nLast Election started 2135194 ms ago, reason: timeout\nLast Election won: 2135188 ms ago\nElection timer: 5000\nLog: [135, 135]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: &lt;-d64b -&gt;d64b &lt;-4984 -&gt;4984\nDisconnections: 0\nServers:\n    4984 (4984 at tcp:[172.18.0.4]:6643) next_index=135 match_index=134 last msg 1084 ms ago\n    1b9a (1b9a at tcp:[172.18.0.3]:6643) (self) next_index=2 match_index=134\nd64b (d64b at tcp:[172.18.0.5]:6643) next_index=135 match_index=134 last msg 1084 ms ago\nstatus: ok\n</code></pre> <p><code>kube-ovn-control-plane2</code> corresponds to a node IP of <code>172.18.0.5</code> and the corresponding ID within the cluster is <code>d64b</code>. Next, kick the node out of the ovn-nb cluster.</p> <pre><code># kubectl ko nb kick d64b\nstarted removal\n</code></pre> <p>Check if the node has been kicked:</p> <pre><code># kubectl ko nb status\n1b9a\nName: OVN_Northbound\nCluster ID: 32ca (32ca07fb-739b-4257-b510-12fa18e7cce8)\nServer ID: 1b9a (1b9a5d76-e69b-410c-8085-39943d0cd38c)\nAddress: tcp:[172.18.0.3]:6643\nStatus: cluster member\nRole: leader\nTerm: 1\nLeader: self\nVote: self\n\nLast Election started 2297649 ms ago, reason: timeout\nLast Election won: 2297643 ms ago\nElection timer: 5000\nLog: [136, 136]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: &lt;-4984 -&gt;4984\nDisconnections: 2\nServers:\n    4984 (4984 at tcp:[172.18.0.4]:6643) next_index=136 match_index=135 last msg 1270 ms ago\n    1b9a (1b9a at tcp:[172.18.0.3]:6643) (self) next_index=2 match_index=135\nstatus: ok\n</code></pre>"},{"location":"en/ops/change-ovn-central-node/#kick-node-in-ovn-sb","title":"Kick Node in ovn-sb","text":"<p>Next, for the ovn-sb cluster, you need to first check the ID of the node within the cluster for subsequent operations.</p> <pre><code>kubectl ko sb status\n3722\nName: OVN_Southbound\nCluster ID: d4bd (d4bd37a4-0400-499f-b4df-b4fd389780f0)\nServer ID: 3722 (3722d5ae-2ced-4820-a6b2-8b744d11fb3e)\nAddress: tcp:[172.18.0.3]:6644\nStatus: cluster member\nRole: leader\nTerm: 1\nLeader: self\nVote: self\n\nLast Election started 2395317 ms ago, reason: timeout\nLast Election won: 2395316 ms ago\nElection timer: 5000\nLog: [130, 130]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: &lt;-e9f7 -&gt;e9f7 &lt;-6e84 -&gt;6e84\nDisconnections: 0\nServers:\n    e9f7 (e9f7 at tcp:[172.18.0.5]:6644) next_index=130 match_index=129 last msg 1006 ms ago\n    6e84 (6e84 at tcp:[172.18.0.4]:6644) next_index=130 match_index=129 last msg 1004 ms ago\n    3722 (3722 at tcp:[172.18.0.3]:6644) (self) next_index=2 match_index=129\nstatus: ok\n</code></pre> <p><code>kube-ovn-control-plane2</code> corresponds to node IP <code>172.18.0.5</code> and the corresponding ID within the cluster is <code>e9f7</code>. Next, kick the node out of the ovn-sb cluster.</p> <pre><code># kubectl ko sb kick e9f7\nstarted removal\n</code></pre> <p>Check if the node has been kicked:</p> <pre><code># kubectl ko sb status\n3722\nName: OVN_Southbound\nCluster ID: d4bd (d4bd37a4-0400-499f-b4df-b4fd389780f0)\nServer ID: 3722 (3722d5ae-2ced-4820-a6b2-8b744d11fb3e)\nAddress: tcp:[172.18.0.3]:6644\nStatus: cluster member\nRole: leader\nTerm: 1\nLeader: self\nVote: self\n\nLast Election started 2481636 ms ago, reason: timeout\nLast Election won: 2481635 ms ago\nElection timer: 5000\nLog: [131, 131]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: &lt;-6e84 -&gt;6e84\nDisconnections: 2\nServers:\n    6e84 (6e84 at tcp:[172.18.0.4]:6644) next_index=131 match_index=130 last msg 642 ms ago\n    3722 (3722 at tcp:[172.18.0.3]:6644) (self) next_index=2 match_index=130\nstatus: ok\n</code></pre>"},{"location":"en/ops/change-ovn-central-node/#delete-node-label-and-downscale-ovn-central","title":"Delete Node Label and Downscale ovn-central","text":"<p>Note that you need to remove the offline node from the node address of the ovn-central environment variable <code>NODE_IPS</code>.</p> <pre><code>kubectl label node kube-ovn-control-plane2 kube-ovn/role-\nkubectl scale deployment -n kube-system ovn-central --replicas=2\nkubectl set env deployment/ovn-central -n kube-system NODE_IPS=\"172.18.0.3,172.18.0.4\"\nkubectl rollout status deployment/ovn-central -n kube-system </code></pre>"},{"location":"en/ops/change-ovn-central-node/#modify-components-address-to-ovn-central","title":"Modify Components Address to ovn-central","text":"<p>Modify <code>ovs-ovn</code> to remove the offline Node address:</p> <pre><code># kubectl set env daemonset/ovs-ovn -n kube-system OVN_DB_IPS=\"172.18.0.3,172.18.0.4\"\ndaemonset.apps/ovs-ovn env updated\n# kubectl delete pod -n kube-system -lapp=ovs\npod \"ovs-ovn-4f6jc\" deleted\npod \"ovs-ovn-csn2w\" deleted\npod \"ovs-ovn-mpbmb\" deleted\n</code></pre> <p>Modify <code>kube-ovn-controller</code> to remove the offline Node address:</p> <pre><code># kubectl set env deployment/kube-ovn-controller -n kube-system OVN_DB_IPS=\"172.18.0.3,172.18.0.4\"\ndeployment.apps/kube-ovn-controller env updated\n\n# kubectl rollout status deployment/kube-ovn-controller -n kube-system\nWaiting for deployment \"kube-ovn-controller\" rollout to finish: 1 of 3 updated replicas are available...\nWaiting for deployment \"kube-ovn-controller\" rollout to finish: 2 of 3 updated replicas are available...\ndeployment \"kube-ovn-controller\" successfully rolled out\n</code></pre>"},{"location":"en/ops/change-ovn-central-node/#clean-node","title":"Clean Node","text":"<p>Delete the database files in the <code>kube-ovn-control-plane2</code> node to avoid errors when adding the node again:</p> <pre><code>rm -rf /etc/origin/ovn\n</code></pre> <p>To take a node offline from a Kubernetes cluster entirely, please continue with Delete Work Node.</p>"},{"location":"en/ops/change-ovn-central-node/#ovn-central-online","title":"ovn-central Online","text":"<p>The following steps will add a new Kubernetes node to the <code>ovn-central</code> cluster.</p>"},{"location":"en/ops/change-ovn-central-node/#directory-check","title":"Directory Check","text":"<p>Check if the <code>ovnnb_db.db</code> or <code>ovnsb_db.db</code> file exists in the <code>/etc/origin/ovn</code> directory of the new node, and if so, delete it:</p> <pre><code>rm -rf /etc/origin/ovn\n</code></pre>"},{"location":"en/ops/change-ovn-central-node/#check-current-ovn-central-status","title":"Check Current ovn-central Status","text":"<p>If the current <code>ovn-central</code> cluster state is already abnormal, adding new nodes may cause the voting election to fail to pass the majority, affecting subsequent operations.</p> <pre><code># kubectl ko nb status\n1b9a\nName: OVN_Northbound\nCluster ID: 32ca (32ca07fb-739b-4257-b510-12fa18e7cce8)\nServer ID: 1b9a (1b9a5d76-e69b-410c-8085-39943d0cd38c)\nAddress: tcp:[172.18.0.3]:6643\nStatus: cluster member\nRole: leader\nTerm: 44\nLeader: self\nVote: self\n\nLast Election started 1855739 ms ago, reason: timeout\nLast Election won: 1855729 ms ago\nElection timer: 5000\nLog: [147, 147]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: -&gt;4984 &lt;-4984\nDisconnections: 0\nServers:\n    4984 (4984 at tcp:[172.18.0.4]:6643) next_index=147 match_index=146 last msg 367 ms ago\n    1b9a (1b9a at tcp:[172.18.0.3]:6643) (self) next_index=140 match_index=146\nstatus: ok\n\n# kubectl ko sb status\n3722\nName: OVN_Southbound\nCluster ID: d4bd (d4bd37a4-0400-499f-b4df-b4fd389780f0)\nServer ID: 3722 (3722d5ae-2ced-4820-a6b2-8b744d11fb3e)\nAddress: tcp:[172.18.0.3]:6644\nStatus: cluster member\nRole: leader\nTerm: 33\nLeader: self\nVote: self\n\nLast Election started 1868589 ms ago, reason: timeout\nLast Election won: 1868579 ms ago\nElection timer: 5000\nLog: [142, 142]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: -&gt;6e84 &lt;-6e84\nDisconnections: 0\nServers:\n    6e84 (6e84 at tcp:[172.18.0.4]:6644) next_index=142 match_index=141 last msg 728 ms ago\n    3722 (3722 at tcp:[172.18.0.3]:6644) (self) next_index=134 match_index=141\nstatus: ok\n</code></pre>"},{"location":"en/ops/change-ovn-central-node/#label-node-and-scale-ovn-central","title":"Label Node and Scale ovn-central","text":"<p>Note that you need to add the online node address to the node address of the ovn-central environment variable <code>NODE_IPS</code>.</p> <pre><code>kubectl label node kube-ovn-control-plane2 kube-ovn/role=master\nkubectl scale deployment -n kube-system ovn-central --replicas=3\nkubectl set env deployment/ovn-central -n kube-system NODE_IPS=\"172.18.0.3,172.18.0.4,172.18.0.5\"\nkubectl rollout status deployment/ovn-central -n kube-system\n</code></pre>"},{"location":"en/ops/change-ovn-central-node/#modify-components-address-to-ovn-central_1","title":"Modify Components Address to ovn-central","text":"<p>Modify <code>ovs-ovn</code> to add the online Node address:</p> <pre><code># kubectl set env daemonset/ovs-ovn -n kube-system OVN_DB_IPS=\"172.18.0.3,172.18.0.4,172.18.0.5\"\ndaemonset.apps/ovs-ovn env updated\n# kubectl delete pod -n kube-system -lapp=ovs\npod \"ovs-ovn-4f6jc\" deleted\npod \"ovs-ovn-csn2w\" deleted\npod \"ovs-ovn-mpbmb\" deleted\n</code></pre> <p>Modify <code>kube-ovn-controller</code> to add the online Node address:</p> <pre><code># kubectl set env deployment/kube-ovn-controller -n kube-system OVN_DB_IPS=\"172.18.0.3,172.18.0.4,172.18.0.5\"\ndeployment.apps/kube-ovn-controller env updated\n\n# kubectl rollout status deployment/kube-ovn-controller -n kube-system\nWaiting for deployment \"kube-ovn-controller\" rollout to finish: 1 of 3 updated replicas are available...\nWaiting for deployment \"kube-ovn-controller\" rollout to finish: 2 of 3 updated replicas are available...\ndeployment \"kube-ovn-controller\" successfully rolled out\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/ops/delete-worker-node/","title":"Delete Work Node","text":"<p>If the node is simply removed from Kubernetes, the <code>ovn-controller</code> process running in <code>ovs-ovn</code> on the node will periodically connect to <code>ovn-central</code> to register relevant network information. This leads to additional resource waste and potential rule conflict risk\u3002 Therefore, when removing nodes from within Kubernetes, follow the steps below to ensure that related resources are cleaned up properly.</p> <p>This document describes the steps to delete a worker node, if you want to change the node where <code>ovn-central</code> is located, please refer to Replace ovn-central Node.</p>"},{"location":"en/ops/delete-worker-node/#evict-pods-on-the-node","title":"Evict Pods on the Node","text":"<pre><code> # kubectl drain kube-ovn-worker --ignore-daemonsets --force\nnode/kube-ovn-worker cordoned\n WARNING: ignoring DaemonSet-managed Pods: kube-system/kube-ovn-cni-zt74b, kube-system/kube-ovn-pinger-5rxfs, kube-system/kube-proxy-jpmnm, kube-system/ovs-ovn-v2kll\n evicting pod kube-system/coredns-64897985d-qsgpt\n evicting pod local-path-storage/local-path-provisioner-5ddd94ff66-llss6\n evicting pod kube-system/kube-ovn-controller-8459db5ff4-94lxb\n pod/kube-ovn-controller-8459db5ff4-94lxb evicted\n pod/coredns-64897985d-qsgpt evicted\n pod/local-path-provisioner-5ddd94ff66-llss6 evicted\n node/kube-ovn-worker drained\n</code></pre>"},{"location":"en/ops/delete-worker-node/#stop-kubelet-and-docker","title":"Stop kubelet and docker","text":"<p>This step stops the <code>ovs-ovn</code> container to avoid registering information to <code>ovn-central</code>. Log into to the corresponding node and ruu the following commands:</p> <pre><code>systemctl stop kubelet\nsystemctl stop docker\n</code></pre> <p>If using containerd as the CRI, the following command needs to be executed to stop the <code>ovs-ovn</code> container:</p> <pre><code>crictl rm -f $(crictl ps | grep openvswitch | awk '{print $1}')\n</code></pre>"},{"location":"en/ops/delete-worker-node/#cleanup-files-on-node","title":"Cleanup Files on Node","text":"<pre><code>rm -rf /var/run/openvswitch\nrm -rf /var/run/ovn\nrm -rf /etc/origin/openvswitch/\nrm -rf /etc/origin/ovn/\nrm -rf /etc/cni/net.d/00-kube-ovn.conflist\nrm -rf /etc/cni/net.d/01-kube-ovn.conflist\nrm -rf /var/log/openvswitch\nrm -rf /var/log/ovn\n</code></pre>"},{"location":"en/ops/delete-worker-node/#delete-the-node","title":"Delete the Node","text":"<pre><code>kubectl delete no kube-ovn-01\n</code></pre>"},{"location":"en/ops/delete-worker-node/#check-if-node-removed-from-ovn-sb","title":"Check If Node Removed from OVN-SB","text":"<p>In the example below, the node <code>kube-ovn-worker</code> is not removed:</p> <pre><code># kubectl ko sbctl show\nChassis \"b0564934-5a0d-4804-a4c0-476c93596a17\"\nhostname: kube-ovn-worker\n  Encap geneve\n      ip: \"172.18.0.2\"\noptions: {csum=\"true\"}\nPort_Binding kube-ovn-pinger-5rxfs.kube-system\nChassis \"6a29de7e-d731-4eaf-bacd-2f239ee52b28\"\nhostname: kube-ovn-control-plane\n  Encap geneve\n      ip: \"172.18.0.3\"\noptions: {csum=\"true\"}\nPort_Binding coredns-64897985d-nbfln.kube-system\n  Port_Binding node-kube-ovn-control-plane\n  Port_Binding local-path-provisioner-5ddd94ff66-h4tn9.local-path-storage\n  Port_Binding kube-ovn-pinger-hf2p6.kube-system\n  Port_Binding coredns-64897985d-fhwlw.kube-system\n</code></pre>"},{"location":"en/ops/delete-worker-node/#delete-the-chassis-manually","title":"Delete the Chassis Manually","text":"<p>Use the uuid find above to delete the chassis:</p> <pre><code># kubectl ko sbctl chassis-del b0564934-5a0d-4804-a4c0-476c93596a17\n# kubectl ko sbctl show\nChassis \"6a29de7e-d731-4eaf-bacd-2f239ee52b28\"\nhostname: kube-ovn-control-plane\n  Encap geneve\n      ip: \"172.18.0.3\"\noptions: {csum=\"true\"}\nPort_Binding coredns-64897985d-nbfln.kube-system\n  Port_Binding node-kube-ovn-control-plane\n  Port_Binding local-path-provisioner-5ddd94ff66-h4tn9.local-path-storage\n  Port_Binding kube-ovn-pinger-hf2p6.kube-system\n  Port_Binding coredns-64897985d-fhwlw.kube-system\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/ops/faq/","title":"FAQ","text":""},{"location":"en/ops/faq/#kylin-arm-system-cross-host-container-access-intermittently-fails","title":"Kylin ARM system cross-host container access intermittently fails","text":""},{"location":"en/ops/faq/#behavior","title":"Behavior","text":"<p>There is a problem with Kylin ARM system and some NIC offload, which can cause intermittent container network failure.</p> <p>Use <code>netstat</code> to identify the problem:</p> <pre><code># netstat -us\nIcmpMsg:\n    InType0: 22\nInType3: 24\nInType8: 117852\nOutType0: 117852\nOutType3: 29\nOutType8: 22\nUdp:\n    3040636 packets received\n    0 packets to unknown port received.\n    4 packet receive errors\n    602 packets sent\n    0 receive buffer errors\n    0 send buffer errors\n    InCsumErrors: 4\nUdpLite:\nIpExt:\n    InBcastPkts: 10244\nInOctets: 4446320361\nOutOctets: 1496815600\nInBcastOctets: 3095950\nInNoECTPkts: 7683903\n</code></pre> <p>If <code>InCsumErrors</code> is present and increases with netwoork failures, you can confirm that this is the problem.</p>"},{"location":"en/ops/faq/#solution","title":"Solution","text":"<p>The fundamental solution requires communication with Kylin and the corresponding network card manufacturer to update the system and drivers. A temporary solution would be to turn off <code>tx offload</code> on the physical NIC, but this would cause a significant degradation in tcp performance.</p> <pre><code>ethtool -K eth0 tx off\n</code></pre> <p>From the community feedback, the problem can be solved by the <code>4.19.90-25.16.v2101</code> kernel.</p>"},{"location":"en/ops/faq/#pod-can-not-access-service","title":"Pod can not Access Service","text":""},{"location":"en/ops/faq/#behavior_1","title":"Behavior","text":"<p>Pod can not access Service, and <code>dmesg</code> show errors:</p> <pre><code>netlink\uff1aUnknown conntrack attr (type=6, max=5)\nopenvswitch: netlink: Flow actions may not be safe on all matching packets.\n</code></pre> <p>This log indicates that the in-kernel OVS version is too low to support the corresponding NAT operation.</p>"},{"location":"en/ops/faq/#solution_1","title":"Solution","text":"<ol> <li>Upgrade the kernel module or compile the OVS kernel module manually.</li> <li>If you are using an Overlay network you can change the <code>kube-ovn-controller</code> args, setting <code>--enable-lb=false</code> to disable the OVN LB to use kube-proxy for service forwarding.</li> </ol> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/ops/from-calico/","title":"Install Kube-OVN From Calico","text":"<p>If a Kubernetes cluster already has Calico installed and needs to change to Kube-OVN you can refer to this document.</p> <p>Since the installation of Calico may vary from version to version and the existing Pod network may be disrupted during the replacement process, it is recommended that you plan ahead and compare the differences in Calico installation from version to version.</p>"},{"location":"en/ops/from-calico/#uninstall-calico","title":"Uninstall Calico","text":"<p>For Calico installed from an Operator:</p> <pre><code>kubectl delete -f https://projectcalico.docs.tigera.io/manifests/tigera-operator.yaml\nkubectl delete -f https://projectcalico.docs.tigera.io/manifests/custom-resources.yaml\n</code></pre> <p>For Calico installed from manifests:</p> <pre><code>kubectl delete -f https://projectcalico.docs.tigera.io/manifests/calico.yaml </code></pre>"},{"location":"en/ops/from-calico/#cleanup-config-files","title":"Cleanup Config Files","text":"<p>Delete the CNI-related configuration files on each machine, depending on the environment:</p> <pre><code>rm -f /etc/cni/net.d/10-calico.conflist\nrm -f /etc/cni/net.d/calico-kubeconfig\n</code></pre> <p>Calico still leaves routing rules, iptables rules, veth network interfaces and other configuration information on the node, so it is recommended to reboot the node to clean up the relevant configuration to avoid problems that are difficult to troubleshoot.</p>"},{"location":"en/ops/from-calico/#install-kube-ovn","title":"Install Kube-OVN","text":"<p>You can refer to One Click Installation to install Kube-OVN as usual.</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/ops/kubectl-ko/","title":"Kubectl Plugin","text":"<p>To facilitate daily operations and maintenance, Kube-OVN provides the kubectl plug-in tool, which allows administrators to perform daily operations through this command. For examples: Check OVN database information and status, OVN database backup and restore, OVS related information, tcpdump specific containers, specific link logical topology, network problem diagnosis and performance optimization.</p>"},{"location":"en/ops/kubectl-ko/#plugin-installation","title":"Plugin Installation","text":"<p>Kube-OVN installation will deploy the plugin to each node by default. If the machine that runs kubectl is not in the cluster, or if you need to reinstall the plugin, please refer to the following steps:</p> <p>Download <code>kubectl-ko</code> file:</p> <pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/images/kubectl-ko\n</code></pre> <p>Move file to <code>$PATH</code>:</p> <pre><code>mv kubectl-ko /usr/local/bin/kubectl-ko\n</code></pre> <p>Add executable permissions:</p> <pre><code>chmod +x /usr/local/bin/kubectl-ko\n</code></pre> <p>Check if the plugin works properly:</p> <pre><code># kubectl plugin list\nThe following compatible plugins are available:\n\n/usr/local/bin/kubectl-ko\n</code></pre>"},{"location":"en/ops/kubectl-ko/#plugin-usage","title":"Plugin Usage","text":"<p>Running <code>kubectl ko</code> will show all the available commands and usage descriptions, as follows:</p> <pre><code># kubectl ko\nkubectl ko {subcommand} [option...]\nAvailable Subcommands:\n  [nb|sb] [status|kick|backup|dbstatus|restore]     ovn-db operations show cluster status, kick stale server, backup database, get db consistency status or restore ovn nb db when met 'inconsistent data' error\n  nbctl [ovn-nbctl options ...]    invoke ovn-nbctl\n  sbctl [ovn-sbctl options ...]    invoke ovn-sbctl\n  vsctl {nodeName} [ovs-vsctl options ...]   invoke ovs-vsctl on the specified node\n  ofctl {nodeName} [ovs-ofctl options ...]   invoke ovs-ofctl on the specified node\n  dpctl {nodeName} [ovs-dpctl options ...]   invoke ovs-dpctl on the specified node\n  appctl {nodeName} [ovs-appctl options ...]   invoke ovs-appctl on the specified node\n  tcpdump {namespace/podname} [tcpdump options ...]     capture pod traffic\n  trace {namespace/podname} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp/udp port]    trace ICMP/TCP/UDP\n  trace {namespace/podname} {target ip address} [target mac address] arp {request|reply}                     trace ARP request/reply\n  trace {node//nodename} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp/udp port]       trace ICMP/TCP/UDP\n  trace {node//nodename} {target ip address} [target mac address] arp {request|reply}                        trace ARP request/reply\n  echo \"  diagnose {all|node|subnet|IPPorts} [nodename|subnetName|{proto1}-{IP1}-{Port1},{proto2}-{IP2}-{Port2}]    diagnose connectivity of all nodes or a specific node or specify subnet's ds pod or IPPorts like 'tcp-172.18.0.2-53,udp-172.18.0.3-53'\"\ntuning {install-fastpath|local-install-fastpath|remove-fastpath|install-stt|local-install-stt|remove-stt} {centos7|centos8}} [kernel-devel-version]  deploy  kernel optimisation components to the system\n  reload    restart all kube-ovn components\n  log {kube-ovn|ovn|ovs|linux|all}    save log to ./kubectl-ko-log/\n  perf [image] performance test default image is kubeovn/test:v1.12.0  </code></pre> <p>The specific functions and usage of each command are described below.</p>"},{"location":"en/ops/kubectl-ko/#nb-sb-status-kick-backup-dbstatus-restore","title":"[nb | sb] [status | kick | backup | dbstatus | restore]","text":"<p>This subcommand mainly operates on OVN northbound or southbound databases, including database cluster status check, database node offline, database backup, database storage status check and database repair.</p>"},{"location":"en/ops/kubectl-ko/#db-cluster-status-check","title":"DB Cluster Status Check","text":"<p>This command executes <code>ovs-appctl cluster/status</code> on the leader node of the corresponding OVN database to show the cluster status:</p> <pre><code># kubectl ko nb status\n306b\nName: OVN_Northbound\nCluster ID: 9a87 (9a872522-3e7d-47ca-83a3-d74333e1a7ca)\nServer ID: 306b (306b256b-b5e1-4eb0-be91-4ca96adf6bad)\nAddress: tcp:[172.18.0.2]:6643\nStatus: cluster member\nRole: leader\nTerm: 1\nLeader: self\nVote: self\n\nLast Election started 280309 ms ago, reason: timeout\nLast Election won: 280309 ms ago\nElection timer: 5000\nLog: [139, 139]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: &lt;-8723 -&gt;8723 &lt;-85d6 -&gt;85d6\nDisconnections: 0\nServers:\n    85d6 (85d6 at tcp:[172.18.0.4]:6643) next_index=139 match_index=138 last msg 763 ms ago\n    8723 (8723 at tcp:[172.18.0.3]:6643) next_index=139 match_index=138 last msg 763 ms ago\n    306b (306b at tcp:[172.18.0.2]:6643) (self) next_index=2 match_index=138\nstatus: ok\n</code></pre> <p>If the <code>match_index</code> under <code>Server</code> has a large difference and the <code>last msg</code> time is long, the corresponding Server may not respond for a long time and needs to be checked further.</p>"},{"location":"en/ops/kubectl-ko/#db-nodes-offline","title":"DB Nodes Offline","text":"<p>This command removes a node from the OVN database and is required when a node is taken offline or replaced. The following is an example of the cluster status from the previous command, to offline the <code>172.18.0.3</code> node:</p> <pre><code># kubectl ko nb kick 8723\nstarted removal\n</code></pre> <p>Check the database cluster status again to confirm that the node has been removed:</p> <pre><code># kubectl ko nb status\n306b\nName: OVN_Northbound\nCluster ID: 9a87 (9a872522-3e7d-47ca-83a3-d74333e1a7ca)\nServer ID: 306b (306b256b-b5e1-4eb0-be91-4ca96adf6bad)\nAddress: tcp:[172.18.0.2]:6643\nStatus: cluster member\nRole: leader\nTerm: 1\nLeader: self\nVote: self\n\nLast Election started 324356 ms ago, reason: timeout\nLast Election won: 324356 ms ago\nElection timer: 5000\nLog: [140, 140]\nEntries not yet committed: 0\nEntries not yet applied: 0\nConnections: &lt;-85d6 -&gt;85d6\nDisconnections: 2\nServers:\n    85d6 (85d6 at tcp:[172.18.0.4]:6643) next_index=140 match_index=139 last msg 848 ms ago\n    306b (306b at tcp:[172.18.0.2]:6643) (self) next_index=2 match_index=139\nstatus: ok\n</code></pre>"},{"location":"en/ops/kubectl-ko/#db-backup","title":"DB Backup","text":"<p>This subcommand backs up the current OVN database locally and can be used for disaster recovery:</p> <pre><code># kubectl ko nb backup\ntar: Removing leading `/' from member names\nbackup ovn-nb db to /root/ovnnb_db.060223191654183154.backup\n</code></pre>"},{"location":"en/ops/kubectl-ko/#database-storage-status-check","title":"Database Storage Status Check","text":"<p>This command is used to check if the database file is corrupt:</p> <pre><code># kubectl ko nb dbstatus\nstatus: ok\n</code></pre> <p>If error happens, <code>inconsistent data</code> is displayed and needs to be fixed with the following command.</p>"},{"location":"en/ops/kubectl-ko/#database-repair","title":"Database Repair","text":"<p>If the database status goes to <code>inconsistent data</code>, this command can be used to repair:</p> <pre><code># kubectl ko nb restore\ndeployment.apps/ovn-central scaled\novn-central original replicas is 3\nfirst nodeIP is 172.18.0.5\novs-ovn pod on node 172.18.0.5 is ovs-ovn-8jxv9\novs-ovn pod on node 172.18.0.3 is ovs-ovn-sjzb6\novs-ovn pod on node 172.18.0.4 is ovs-ovn-t87zk\nbackup nb db file\nrestore nb db file, operate in pod ovs-ovn-8jxv9\ndeployment.apps/ovn-central scaled\nfinish restore nb db file and ovn-central replicas\nrecreate ovs-ovn pods\npod \"ovs-ovn-8jxv9\" deleted\npod \"ovs-ovn-sjzb6\" deleted\npod \"ovs-ovn-t87zk\" deleted\n</code></pre>"},{"location":"en/ops/kubectl-ko/#nbctl-sbctl-options","title":"[nbctl | sbctl] [options ...]","text":"<p>This subcommand executes the <code>ovn-nbctl</code> and <code>ovn-sbctl</code> commands directly into the leader node of the OVN northbound or southbound database. For more detailed usage of this command, please refer to the official documentation of the upstream OVN ovn-nbctl(8) \u548c ovn-sbctl(8)\u3002</p> <pre><code># kubectl ko nbctl show\nswitch c7cd17e8-ceee-4a91-9bb3-e5a313fe1ece (snat)\nport snat-ovn-cluster\n        type: router\n        router-port: ovn-cluster-snat\nswitch 20e0c6d0-023a-4756-aec5-200e0c60f95d (join)\nport node-liumengxin-ovn3-192.168.137.178\n        addresses: [\"00:00:00:64:FF:A8 100.64.0.4\"]\nport node-liumengxin-ovn1-192.168.137.176\n        addresses: [\"00:00:00:AF:98:62 100.64.0.2\"]\nport node-liumengxin-ovn2-192.168.137.177\n        addresses: [\"00:00:00:D9:58:B8 100.64.0.3\"]\nport join-ovn-cluster\n        type: router\n        router-port: ovn-cluster-join\nswitch 0191705c-f827-427b-9de3-3c3b7d971ba5 (central)\nport central-ovn-cluster\n        type: router\n        router-port: ovn-cluster-central\nswitch 2a45ff05-388d-4f85-9daf-e6fccd5833dc (ovn-default)\nport alertmanager-main-0.monitoring\n        addresses: [\"00:00:00:6C:DF:A3 10.16.0.19\"]\nport kube-state-metrics-5d6885d89-4nf8h.monitoring\n        addresses: [\"00:00:00:6F:02:1C 10.16.0.15\"]\nport fake-kubelet-67c55dfd89-pv86k.kube-system\n        addresses: [\"00:00:00:5C:12:E8 10.16.19.177\"]\nport ovn-default-ovn-cluster\n        type: router\n        router-port: ovn-cluster-ovn-default\nrouter 212f73dd-d63d-4d72-864b-a537e9afbee1 (ovn-cluster)\nport ovn-cluster-snat\n        mac: \"00:00:00:7A:82:8F\"\nnetworks: [\"172.22.0.1/16\"]\nport ovn-cluster-join\n        mac: \"00:00:00:F8:18:5A\"\nnetworks: [\"100.64.0.1/16\"]\nport ovn-cluster-central\n        mac: \"00:00:00:4D:8C:F5\"\nnetworks: [\"192.101.0.1/16\"]\nport ovn-cluster-ovn-default\n        mac: \"00:00:00:A3:F8:18\"\nnetworks: [\"10.16.0.1/16\"]\n</code></pre>"},{"location":"en/ops/kubectl-ko/#vsctl-nodename-options","title":"vsctl {nodeName} [options ...]","text":"<p>This command will go to the <code>ovs-ovn</code> container on the corresponding <code>nodeName</code> and execute the corresponding <code>ovs-vsctl</code> command to query and configure <code>vswitchd</code>. For more detailed usage of this command, please refer to the official documentation of the upstream OVS ovs-vsctl(8)\u3002</p> <pre><code># kubectl ko vsctl kube-ovn-01 show\n0d4c4675-c9cc-440a-8c1a-878e17f81b88\n    Bridge br-int\n        fail_mode: secure\n        datapath_type: system\n        Port a2c1a8a8b83a_h\n            Interface a2c1a8a8b83a_h\n        Port \"4fa5c4cbb1a5_h\"\nInterface \"4fa5c4cbb1a5_h\"\nPort ovn-eef07d-0\n            Interface ovn-eef07d-0\n                type: stt\n                options: {csum=\"true\", key=flow, remote_ip=\"192.168.137.178\"}\nPort ovn0\n            Interface ovn0\n                type: internal\n        Port mirror0\n            Interface mirror0\n                type: internal\n        Port ovn-efa253-0\n            Interface ovn-efa253-0\n                type: stt\n                options: {csum=\"true\", key=flow, remote_ip=\"192.168.137.177\"}\nPort br-int\n            Interface br-int\n                type: internal\n    ovs_version: \"2.17.2\"\n</code></pre>"},{"location":"en/ops/kubectl-ko/#ofctl-nodename-options","title":"ofctl {nodeName} [options ...]","text":"<p>This command will go to the <code>ovs-ovn</code> container on the corresponding <code>nodeName</code> and execute the corresponding <code>ovs-ofctl</code> command to query or manage OpenFlow. For more detailed usage of this command, please refer to the official documentation of the upstream OVS ovs-ofctl(8)\u3002</p> <pre><code># kubectl ko ofctl kube-ovn-01 dump-flows br-int\nNXST_FLOW reply (xid=0x4): flags=[more]\ncookie=0xcf3429e6, duration=671791.432s, table=0, n_packets=0, n_bytes=0, idle_age=65534, hard_age=65534, priority=100,in_port=2 actions=load:0x4-&gt;NXM_NX_REG13[],load:0x9-&gt;NXM_NX_REG11[],load:0xb-&gt;NXM_NX_REG12[],load:0x4-&gt;OXM_OF_METADATA[],load:0x1-&gt;NXM_NX_REG14[],resubmit(,8)\ncookie=0xc91413c6, duration=671791.431s, table=0, n_packets=907489, n_bytes=99978275, idle_age=0, hard_age=65534, priority=100,in_port=7 actions=load:0x1-&gt;NXM_NX_REG13[],load:0x9-&gt;NXM_NX_REG11[],load:0xb-&gt;NXM_NX_REG12[],load:0x4-&gt;OXM_OF_METADATA[],load:0x4-&gt;NXM_NX_REG14[],resubmit(,8)\ncookie=0xf180459, duration=671791.431s, table=0, n_packets=17348582, n_bytes=2667811214, idle_age=0, hard_age=65534, priority=100,in_port=6317 actions=load:0xa-&gt;NXM_NX_REG13[],load:0x9-&gt;NXM_NX_REG11[],load:0xb-&gt;NXM_NX_REG12[],load:0x4-&gt;OXM_OF_METADATA[],load:0x9-&gt;NXM_NX_REG14[],resubmit(,8)\ncookie=0x7806dd90, duration=671791.431s, table=0, n_packets=3235428, n_bytes=833821312, idle_age=0, hard_age=65534, priority=100,in_port=1 actions=load:0xd-&gt;NXM_NX_REG13[],load:0x9-&gt;NXM_NX_REG11[],load:0xb-&gt;NXM_NX_REG12[],load:0x4-&gt;OXM_OF_METADATA[],load:0x3-&gt;NXM_NX_REG14[],resubmit(,8)\n...\n</code></pre>"},{"location":"en/ops/kubectl-ko/#dpctl-nodename-options","title":"dpctl {nodeName} [options ...]","text":"<p>This command will go to the <code>ovs-ovn</code> container on the corresponding <code>nodeName</code> and execute the corresponding <code>ovs-dpctl</code> command to query or manage the OVS datapath. For more detailed usage of this command, please refer to the official documentation of the upstream OVS ovs-dpctl(8)\u3002</p> <pre><code># kubectl ko dpctl kube-ovn-01 show\nsystem@ovs-system:\n  lookups: hit:350805055 missed:21983648 lost:73\n  flows: 105\nmasks: hit:1970748791 total:22 hit/pkt:5.29\n  port 0: ovs-system (internal)\nport 1: ovn0 (internal)\nport 2: mirror0 (internal)\nport 3: br-int (internal)\nport 4: stt_sys_7471 (stt: packet_type=ptap)\nport 5: eeb4d9e51b5d_h\n  port 6: a2c1a8a8b83a_h\n  port 7: 4fa5c4cbb1a5_h\n</code></pre>"},{"location":"en/ops/kubectl-ko/#appctl-nodename-options","title":"appctl {nodeName} [options ...]","text":"<p>This command will enter the <code>ovs-ovn</code> container on the corresponding <code>nodeName</code> and execute the corresponding <code>ovs-appctl</code> command to operate the associated daemon process. For more detailed usage of this command, please refer to the official documentation of the upstream OVS ovs-appctl(8)\u3002</p> <pre><code># kubectl ko appctl kube-ovn-01 vlog/list\nconsole    syslog    file\n                 -------    ------    ------\nbacktrace          OFF        ERR       INFO\nbfd                OFF        ERR       INFO\nbond               OFF        ERR       INFO\nbridge             OFF        ERR       INFO\nbundle             OFF        ERR       INFO\nbundles            OFF        ERR       INFO\n...\n</code></pre>"},{"location":"en/ops/kubectl-ko/#tcpdump-namespacepodname-tcpdump-options","title":"tcpdump {namespace/podname} [tcpdump options ...]","text":"<p>This command will enter the <code>kube-ovn-cni</code> container on the machine where <code>namespace/podname</code> is located, and run <code>tcpdump</code> to capture the traffic on the veth NIC of the corresponding container, which can be used to troubleshoot network-related problems.</p> <pre><code># kubectl ko tcpdump default/ds1-l6n7p icmp\n+ kubectl exec -it kube-ovn-cni-wlg4s -n kube-ovn -- tcpdump -nn -i d7176fe7b4e0_h icmp\ntcpdump: verbose output suppressed, use -v or -vv for full protocol decode\nlistening on d7176fe7b4e0_h, link-type EN10MB (Ethernet), capture size 262144 bytes\n06:52:36.619688 IP 100.64.0.3 &gt; 10.16.0.4: ICMP echo request, id 2, seq 1, length 64\n06:52:36.619746 IP 10.16.0.4 &gt; 100.64.0.3: ICMP echo reply, id 2, seq 1, length 64\n06:52:37.619588 IP 100.64.0.3 &gt; 10.16.0.4: ICMP echo request, id 2, seq 2, length 64\n06:52:37.619630 IP 10.16.0.4 &gt; 100.64.0.3: ICMP echo reply, id 2, seq 2, length 64\n06:52:38.619933 IP 100.64.0.3 &gt; 10.16.0.4: ICMP echo request, id 2, seq 3, length 64\n06:52:38.619973 IP 10.16.0.4 &gt; 100.64.0.3: ICMP echo reply, id 2, seq 3, length 64\n</code></pre>"},{"location":"en/ops/kubectl-ko/#trace-arguments","title":"trace [arguments ...]","text":"<p>This command will print the OVN logical flow table and the final Openflow flow table when the Pod/node accesses an address through a specific protocol, so that it make locate flow table related problems during development or troubleshooting much easy.</p> <p>Supported commands:</p> <pre><code>kubectl ko trace {namespace/podname} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp/udp port]\nkubectl ko trace {namespace/podname} {target ip address} [target mac address] arp {request|reply}\nkubectl ko trace {node//nodename} {target ip address} [target mac address] {icmp|tcp|udp} [target tcp/udp port]\nkubectl ko trace {node//nodename} {target ip address} [target mac address] arp {request|reply}\n</code></pre> <p>Example:</p> <pre><code># kubectl ko trace default/ds1-l6n7p 8.8.8.8 icmp\n+ kubectl exec ovn-central-5bc494cb5-np9hm -n kube-ovn -- ovn-trace --ct=new ovn-default 'inport == \"ds1-l6n7p.default\" &amp;&amp; ip.ttl == 64 &amp;&amp; icmp &amp;&amp; eth.src == 0a:00:00:10:00:05 &amp;&amp; ip4.src == 10.16.0.4 &amp;&amp; eth.dst == 00:00:00:B8:CA:43 &amp;&amp; ip4.dst == 8.8.8.8'\n# icmp,reg14=0xf,vlan_tci=0x0000,dl_src=0a:00:00:10:00:05,dl_dst=00:00:00:b8:ca:43,nw_src=10.16.0.4,nw_dst=8.8.8.8,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=0,icmp_code=0\n\ningress(dp=\"ovn-default\", inport=\"ds1-l6n7p.default\")\n-----------------------------------------------------\n 0. ls_in_port_sec_l2 (ovn-northd.c:4143): inport == \"ds1-l6n7p.default\" &amp;&amp; eth.src == {0a:00:00:10:00:05}, priority 50, uuid 39453393\nnext;\n1. ls_in_port_sec_ip (ovn-northd.c:2898): inport == \"ds1-l6n7p.default\" &amp;&amp; eth.src == 0a:00:00:10:00:05 &amp;&amp; ip4.src == {10.16.0.4}, priority 90, uuid 81bcd485\n    next;\n3. ls_in_pre_acl (ovn-northd.c:3269): ip, priority 100, uuid 7b4f4971\n    reg0[0] = 1;\nnext;\n5. ls_in_pre_stateful (ovn-northd.c:3396): reg0[0] == 1, priority 100, uuid 36cdd577\n    ct_next;\n\nct_next(ct_state=new|trk)\n-------------------------\n 6. ls_in_acl (ovn-northd.c:3759): ip &amp;&amp; (!ct.est || (ct.est &amp;&amp; ct_label.blocked == 1)), priority 1, uuid 7608af5b\n    reg0[1] = 1;\nnext;\n10. ls_in_stateful (ovn-northd.c:3995): reg0[1] == 1, priority 100, uuid 2aba1b90\n    ct_commit(ct_label=0/0x1);\nnext;\n16. ls_in_l2_lkup (ovn-northd.c:4470): eth.dst == 00:00:00:b8:ca:43, priority 50, uuid 5c9c3c9f\n    outport = \"ovn-default-ovn-cluster\";\noutput;\n\n...\n</code></pre> <p>If the trace object is a virtual machine running  in Underlay network, additional parameters is needed to specify the destination Mac address.</p> <pre><code>kubectl ko trace default/virt-handler-7lvml 8.8.8.8 82:7c:9f:83:8c:01 icmp\n</code></pre>"},{"location":"en/ops/kubectl-ko/#diagnose-allnodesubnetipports-nodenamesubnetnameproto1-ip1-port1proto2-ip2-port2","title":"diagnose {all|node|subnet|IPPorts} [nodename|subnetName|{proto1}-{IP1}-{Port1},{proto2}-{IP2}-{Port2}]","text":"<p>Diagnose the status of cluster network components and go to the corresponding node's <code>kube-ovn-pinger</code> to detect connectivity and network latency from the current node to other nodes and critical services.</p> <pre><code># kubectl ko diagnose all\nswitch c7cd17e8-ceee-4a91-9bb3-e5a313fe1ece (snat)\nport snat-ovn-cluster\n        type: router\n        router-port: ovn-cluster-snat\nswitch 20e0c6d0-023a-4756-aec5-200e0c60f95d (join)\nport node-liumengxin-ovn3-192.168.137.178\n        addresses: [\"00:00:00:64:FF:A8 100.64.0.4\"]\nport node-liumengxin-ovn1-192.168.137.176\n        addresses: [\"00:00:00:AF:98:62 100.64.0.2\"]\nport join-ovn-cluster\n        type: router\n        router-port: ovn-cluster-join\nswitch 0191705c-f827-427b-9de3-3c3b7d971ba5 (central)\nport central-ovn-cluster\n        type: router\n        router-port: ovn-cluster-central\nswitch 2a45ff05-388d-4f85-9daf-e6fccd5833dc (ovn-default)\nport ovn-default-ovn-cluster\n        type: router\n        router-port: ovn-cluster-ovn-default\n    port prometheus-k8s-1.monitoring\n        addresses: [\"00:00:00:AA:37:DF 10.16.0.23\"]\nrouter 212f73dd-d63d-4d72-864b-a537e9afbee1 (ovn-cluster)\nport ovn-cluster-snat\n        mac: \"00:00:00:7A:82:8F\"\nnetworks: [\"172.22.0.1/16\"]\nport ovn-cluster-join\n        mac: \"00:00:00:F8:18:5A\"\nnetworks: [\"100.64.0.1/16\"]\nport ovn-cluster-central\n        mac: \"00:00:00:4D:8C:F5\"\nnetworks: [\"192.101.0.1/16\"]\nport ovn-cluster-ovn-default\n        mac: \"00:00:00:A3:F8:18\"\nnetworks: [\"10.16.0.1/16\"]\nRouting Policies\n     31000                            ip4.dst == 10.16.0.0/16           allow\n     31000                           ip4.dst == 100.64.0.0/16           allow\n     30000                         ip4.dst == 192.168.137.177         reroute                100.64.0.3\n     30000                         ip4.dst == 192.168.137.178         reroute                100.64.0.4\n     29000                 ip4.src == $ovn.default.fake.6_ip4         reroute               100.64.0.22\n     29000                 ip4.src == $ovn.default.fake.7_ip4         reroute               100.64.0.21\n     29000                 ip4.src == $ovn.default.fake.8_ip4         reroute               100.64.0.23\n     29000 ip4.src == $ovn.default.liumengxin.ovn3.192.168.137.178_ip4         reroute                100.64.0.4\n     20000 ip4.src == $ovn.default.liumengxin.ovn1.192.168.137.176_ip4 &amp;&amp; ip4.dst != $ovn.cluster.overlay.subnets.IPv4         reroute                100.64.0.2\n     20000 ip4.src == $ovn.default.liumengxin.ovn2.192.168.137.177_ip4 &amp;&amp; ip4.dst != $ovn.cluster.overlay.subnets.IPv4         reroute                100.64.0.3\n     20000 ip4.src == $ovn.default.liumengxin.ovn3.192.168.137.178_ip4 &amp;&amp; ip4.dst != $ovn.cluster.overlay.subnets.IPv4         reroute                100.64.0.4\nIPv4 Routes\nRoute Table &lt;main&gt;:\n                0.0.0.0/0                100.64.0.1 dst-ip\nUUID                                    LB                  PROTO      VIP                     IPs\ne9bcfd9d-793e-4431-9073-6dec96b75d71    cluster-tcp-load    tcp        10.100.209.132:10660    192.168.137.176:10660\n                                                            tcp        10.101.239.192:6641     192.168.137.177:6641\n                                                            tcp        10.101.240.101:3000     10.16.0.7:3000\n                                                            tcp        10.103.184.186:6642     192.168.137.177:6642\n35d2b7a5-e3a7-485a-a4b7-b4970eb0e63b    cluster-tcp-sess    tcp        10.100.158.128:8080     10.16.0.10:8080,10.16.0.5:8080,10.16.63.30:8080\n                                                            tcp        10.107.26.215:8080      10.16.0.19:8080,10.16.0.20:8080,10.16.0.21:8080\n                                                            tcp        10.107.26.215:9093      10.16.0.19:9093,10.16.0.20:9093,10.16.0.21:9093\n                                                            tcp        10.98.187.99:8080       10.16.0.22:8080,10.16.0.23:8080\n                                                            tcp        10.98.187.99:9090       10.16.0.22:9090,10.16.0.23:9090\nf43303e4-89aa-4d3e-a3dc-278a552fe27b    cluster-udp-load    udp        10.96.0.10:53           10.16.0.4:53,10.16.0.9:53\n_uuid               : 06776304-5a96-43ed-90c4-c4854c251699\naddresses           : []\nexternal_ids        : {vendor=kube-ovn}\nname                : node_liumengxin_ovn2_192.168.137.177_underlay_v6\n\n_uuid               : 62690625-87d5-491c-8675-9fd83b1f433c\naddresses           : []\nexternal_ids        : {vendor=kube-ovn}\nname                : node_liumengxin_ovn1_192.168.137.176_underlay_v6\n\n_uuid               : b03a9bae-94d5-4562-b34c-b5f6198e180b\naddresses           : [\"10.16.0.0/16\", \"100.64.0.0/16\", \"172.22.0.0/16\", \"192.101.0.0/16\"]\nexternal_ids        : {vendor=kube-ovn}\nname                : ovn.cluster.overlay.subnets.IPv4\n\n_uuid               : e1056f3a-24cc-4666-8a91-75ee6c3c2426\naddresses           : []\nexternal_ids        : {vendor=kube-ovn}\nname                : ovn.cluster.overlay.subnets.IPv6\n\n_uuid               : 3e5d5fff-e670-47b2-a2f5-a39f4698a8c5\naddresses           : []\nexternal_ids        : {vendor=kube-ovn}\nname                : node_liumengxin_ovn3_192.168.137.178_underlay_v6\n_uuid               : 2d85dbdc-d0db-4abe-b19e-cc806d32b492\naction              : drop\ndirection           : from-lport\nexternal_ids        : {}\nlabel               : 0\nlog                 : false\nmatch               : \"inport==@ovn.sg.kubeovn_deny_all &amp;&amp; ip\"\nmeter               : []\nname                : []\noptions             : {}\npriority            : 2003\nseverity            : []\n\n_uuid               : de790cc8-f155-405f-bb32-5a51f30c545f\naction              : drop\ndirection           : to-lport\nexternal_ids        : {}\nlabel               : 0\nlog                 : false\nmatch               : \"outport==@ovn.sg.kubeovn_deny_all &amp;&amp; ip\"\nmeter               : []\nname                : []\noptions             : {}\npriority            : 2003\nseverity            : []\nChassis \"e15ed4d4-1780-4d50-b09e-ea8372ed48b8\"\nhostname: liumengxin-ovn1-192.168.137.176\n    Encap stt\n        ip: \"192.168.137.176\"\noptions: {csum=\"true\"}\nPort_Binding node-liumengxin-ovn1-192.168.137.176\n    Port_Binding perf-6vxkn.default\n    Port_Binding kube-state-metrics-5d6885d89-4nf8h.monitoring\n    Port_Binding alertmanager-main-0.monitoring\n    Port_Binding kube-ovn-pinger-6ftdf.kube-system\n    Port_Binding fake-kubelet-67c55dfd89-pv86k.kube-system\n    Port_Binding prometheus-k8s-0.monitoring\nChassis \"eef07da1-f8ad-4775-b14d-bd6a3b4eb0d5\"\nhostname: liumengxin-ovn3-192.168.137.178\n    Encap stt\n        ip: \"192.168.137.178\"\noptions: {csum=\"true\"}\nPort_Binding kube-ovn-pinger-7twb4.kube-system\n    Port_Binding prometheus-adapter-86df476d87-rl88g.monitoring\n    Port_Binding prometheus-k8s-1.monitoring\n    Port_Binding node-liumengxin-ovn3-192.168.137.178\n    Port_Binding perf-ff475.default\n    Port_Binding alertmanager-main-1.monitoring\n    Port_Binding blackbox-exporter-676d976865-tvsjd.monitoring\nChassis \"efa253c9-494d-4719-83ae-b48ab0f11c03\"\nhostname: liumengxin-ovn2-192.168.137.177\n    Encap stt\n        ip: \"192.168.137.177\"\noptions: {csum=\"true\"}\nPort_Binding grafana-6c4c6b8fb7-pzd2c.monitoring\n    Port_Binding node-liumengxin-ovn2-192.168.137.177\n    Port_Binding alertmanager-main-2.monitoring\n    Port_Binding coredns-6789c94dd8-9jqsz.kube-system\n    Port_Binding coredns-6789c94dd8-25d4r.kube-system\n    Port_Binding prometheus-operator-7bbc99fc8b-wgjm4.monitoring\n    Port_Binding prometheus-adapter-86df476d87-gdxmc.monitoring\n    Port_Binding perf-fjnws.default\n    Port_Binding kube-ovn-pinger-vh2xg.kube-system\nds kube-proxy ready\nkube-proxy ready\ndeployment ovn-central ready\ndeployment kube-ovn-controller ready\nds kube-ovn-cni ready\nds ovs-ovn ready\ndeployment coredns ready\novn-nb leader check ok\novn-sb leader check ok\novn-northd leader check ok\n### kube-ovn-controller recent log\n\n### start to diagnose node liumengxin-ovn1-192.168.137.176\n#### ovn-controller log:\n2022-06-03T00:56:44.897Z|16722|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T01:06:44.912Z|16723|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T01:16:44.925Z|16724|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T01:26:44.936Z|16725|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T01:36:44.959Z|16726|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T01:46:44.974Z|16727|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T01:56:44.988Z|16728|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T02:06:45.001Z|16729|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T02:16:45.025Z|16730|inc_proc_eng|INFO|User triggered force recompute.\n2022-06-03T02:26:45.040Z|16731|inc_proc_eng|INFO|User triggered force recompute.\n\n#### ovs-vswitchd log:\n2022-06-02T23:03:00.137Z|00079|dpif(handler1)|WARN|system@ovs-system: execute ct(commit,zone=14,label=0/0x1,nat(src)),8 failed (Invalid argument) on packet icmp,vlan_tci=0x0000,dl_src=00:00:00:f8:07:c8,dl_dst=00:00:00:fa:1e:50,nw_src=10.16.0.5,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0 icmp_csum:f9d1\n with metadata skb_priority(0),tunnel(tun_id=0x160017000004,src=192.168.137.177,dst=192.168.137.176,ttl=64,tp_src=38881,tp_dst=7471,flags(csum|key)),skb_mark(0),ct_state(0x21),ct_zone(0xe),ct_tuple4(src=10.16.0.5,dst=10.16.0.10,proto=1,tp_src=8,tp_dst=0),in_port(4) mtu 0\n2022-06-02T23:23:31.840Z|00080|dpif(handler1)|WARN|system@ovs-system: execute ct(commit,zone=14,label=0/0x1,nat(src)),8 failed (Invalid argument) on packet icmp,vlan_tci=0x0000,dl_src=00:00:00:f8:07:c8,dl_dst=00:00:00:fa:1e:50,nw_src=10.16.0.5,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0 icmp_csum:15b2\n with metadata skb_priority(0),tunnel(tun_id=0x160017000004,src=192.168.137.177,dst=192.168.137.176,ttl=64,tp_src=38881,tp_dst=7471,flags(csum|key)),skb_mark(0),ct_state(0x21),ct_zone(0xe),ct_tuple4(src=10.16.0.5,dst=10.16.0.10,proto=1,tp_src=8,tp_dst=0),in_port(4) mtu 0\n2022-06-03T00:09:15.659Z|00081|dpif(handler1)|WARN|system@ovs-system: execute ct(commit,zone=14,label=0/0x1,nat(src)),8 failed (Invalid argument) on packet icmp,vlan_tci=0x0000,dl_src=00:00:00:dc:e3:63,dl_dst=00:00:00:fa:1e:50,nw_src=10.16.63.30,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0 icmp_csum:e5a5\n with metadata skb_priority(0),tunnel(tun_id=0x150017000004,src=192.168.137.178,dst=192.168.137.176,ttl=64,tp_src=9239,tp_dst=7471,flags(csum|key)),skb_mark(0),ct_state(0x21),ct_zone(0xe),ct_tuple4(src=10.16.63.30,dst=10.16.0.10,proto=1,tp_src=8,tp_dst=0),in_port(4) mtu 0\n2022-06-03T00:30:13.409Z|00064|dpif(handler2)|WARN|system@ovs-system: execute ct(commit,zone=14,label=0/0x1,nat(src)),8 failed (Invalid argument) on packet icmp,vlan_tci=0x0000,dl_src=00:00:00:f8:07:c8,dl_dst=00:00:00:fa:1e:50,nw_src=10.16.0.5,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0 icmp_csum:6b4a\n with metadata skb_priority(0),tunnel(tun_id=0x160017000004,src=192.168.137.177,dst=192.168.137.176,ttl=64,tp_src=38881,tp_dst=7471,flags(csum|key)),skb_mark(0),ct_state(0x21),ct_zone(0xe),ct_tuple4(src=10.16.0.5,dst=10.16.0.10,proto=1,tp_src=8,tp_dst=0),in_port(4) mtu 0\n2022-06-03T02:02:33.832Z|00082|dpif(handler1)|WARN|system@ovs-system: execute ct(commit,zone=14,label=0/0x1,nat(src)),8 failed (Invalid argument) on packet icmp,vlan_tci=0x0000,dl_src=00:00:00:f8:07:c8,dl_dst=00:00:00:fa:1e:50,nw_src=10.16.0.5,nw_dst=10.16.0.10,nw_tos=0,nw_ecn=0,nw_ttl=64,icmp_type=8,icmp_code=0 icmp_csum:a819\n with metadata skb_priority(0),tunnel(tun_id=0x160017000004,src=192.168.137.177,dst=192.168.137.176,ttl=64,tp_src=38881,tp_dst=7471,flags(csum|key)),skb_mark(0),ct_state(0x21),ct_zone(0xe),ct_tuple4(src=10.16.0.5,dst=10.16.0.10,proto=1,tp_src=8,tp_dst=0),in_port(4) mtu 0\n\n#### ovs-vsctl show results:\n0d4c4675-c9cc-440a-8c1a-878e17f81b88\n    Bridge br-int\n        fail_mode: secure\n        datapath_type: system\n        Port a2c1a8a8b83a_h\n            Interface a2c1a8a8b83a_h\n        Port \"4fa5c4cbb1a5_h\"\nInterface \"4fa5c4cbb1a5_h\"\nPort ovn-eef07d-0\n            Interface ovn-eef07d-0\n                type: stt\n                options: {csum=\"true\", key=flow, remote_ip=\"192.168.137.178\"}\nPort ovn0\n            Interface ovn0\n                type: internal\n        Port \"04d03360e9a0_h\"\nInterface \"04d03360e9a0_h\"\nPort eeb4d9e51b5d_h\n            Interface eeb4d9e51b5d_h\n        Port mirror0\n            Interface mirror0\n                type: internal\n        Port \"8e5d887ccd80_h\"\nInterface \"8e5d887ccd80_h\"\nPort ovn-efa253-0\n            Interface ovn-efa253-0\n                type: stt\n                options: {csum=\"true\", key=flow, remote_ip=\"192.168.137.177\"}\nPort \"17512d5be1f1_h\"\nInterface \"17512d5be1f1_h\"\nPort br-int\n            Interface br-int\n                type: internal\n    ovs_version: \"2.17.2\"\n\n#### pinger diagnose results:\nI0603 10:35:04.349404   17619 pinger.go:19]\n-------------------------------------------------------------------------------\nKube-OVN:\n  Version:       v1.13.0\n  Build:         2022-04-24_08:02:50\n  Commit:        git-73f9d15\n  Go Version:    go1.17.8\n  Arch:          amd64\n-------------------------------------------------------------------------------\nI0603 10:35:04.376797   17619 config.go:166] pinger config is &amp;{KubeConfigFile: KubeClient:0xc000493380 Port:8080 DaemonSetNamespace:kube-system DaemonSetName:kube-ovn-pinger Interval:5 Mode:job ExitCode:0 InternalDNS:kubernetes.default ExternalDNS: NodeName:liumengxin-ovn1-192.168.137.176 HostIP:192.168.137.176 PodName:kube-ovn-pinger-6ftdf PodIP:10.16.0.10 PodProtocols:[IPv4] ExternalAddress: NetworkMode:kube-ovn PollTimeout:2 PollInterval:15 SystemRunDir:/var/run/openvswitch DatabaseVswitchName:Open_vSwitch DatabaseVswitchSocketRemote:unix:/var/run/openvswitch/db.sock DatabaseVswitchFileDataPath:/etc/openvswitch/conf.db DatabaseVswitchFileLogPath:/var/log/openvswitch/ovsdb-server.log DatabaseVswitchFilePidPath:/var/run/openvswitch/ovsdb-server.pid DatabaseVswitchFileSystemIDPath:/etc/openvswitch/system-id.conf ServiceVswitchdFileLogPath:/var/log/openvswitch/ovs-vswitchd.log ServiceVswitchdFilePidPath:/var/run/openvswitch/ovs-vswitchd.pid ServiceOvnControllerFileLogPath:/var/log/ovn/ovn-controller.log ServiceOvnControllerFilePidPath:/var/run/ovn/ovn-controller.pid}\nI0603 10:35:04.449166   17619 exporter.go:75] liumengxin-ovn1-192.168.137.176: exporter connect successfully\nI0603 10:35:04.554011   17619 ovn.go:21] ovs-vswitchd and ovsdb are up\nI0603 10:35:04.651293   17619 ovn.go:33] ovn_controller is up\nI0603 10:35:04.651342   17619 ovn.go:39] start to check port binding\nI0603 10:35:04.749613   17619 ovn.go:135] chassis id is 1d7f3d6c-eec5-4b3c-adca-2969d9cdfd80\nI0603 10:35:04.763487   17619 ovn.go:49] port in sb is [node-liumengxin-ovn1-192.168.137.176 perf-6vxkn.default kube-state-metrics-5d6885d89-4nf8h.monitoring alertmanager-main-0.monitoring kube-ovn-pinger-6ftdf.kube-system fake-kubelet-67c55dfd89-pv86k.kube-system prometheus-k8s-0.monitoring]\nI0603 10:35:04.763583   17619 ovn.go:61] ovs and ovn-sb binding check passed\nI0603 10:35:05.049309   17619 ping.go:259] start to check apiserver connectivity\nI0603 10:35:05.053666   17619 ping.go:268] connect to apiserver success in 4.27ms\nI0603 10:35:05.053786   17619 ping.go:129] start to check pod connectivity\nI0603 10:35:05.249590   17619 ping.go:159] ping pod: kube-ovn-pinger-6ftdf 10.16.0.10, count: 3, loss count 0, average rtt 16.30ms\nI0603 10:35:05.354135   17619 ping.go:159] ping pod: kube-ovn-pinger-7twb4 10.16.63.30, count: 3, loss count 0, average rtt 1.81ms\nI0603 10:35:05.458460   17619 ping.go:159] ping pod: kube-ovn-pinger-vh2xg 10.16.0.5, count: 3, loss count 0, average rtt 1.92ms\nI0603 10:35:05.458523   17619 ping.go:83] start to check node connectivity\n</code></pre> <p>If the target of diagnose is specified as subnet, the script will create a daemonset on the subnet, and <code>kube-ovn-pinger</code> will detect the connectivity and network delay of all pods in this daemonset, and automatically destroy the daemonset after the test.</p> <p>If the target of diagnose is specified as IPPorts, the script will let each <code>kube-ovn-pinger</code> pod detect whether the target protocol, IP, and Port are reachable.</p>"},{"location":"en/ops/kubectl-ko/#tuning-install-fastpathlocal-install-fastpathremove-fastpathinstall-sttlocal-install-sttremove-stt-centos7centos8-kernel-devel-version","title":"tuning {install-fastpath|local-install-fastpath|remove-fastpath|install-stt|local-install-stt|remove-stt} {centos7|centos8}} [kernel-devel-version]","text":"<p>This command performs performance tuning related operations, please refer to Performance Tunning.</p>"},{"location":"en/ops/kubectl-ko/#reload","title":"reload","text":"<p>This command restarts all Kube-OVN related components:</p> <pre><code># kubectl ko reload\npod \"ovn-central-8684dd94bd-vzgcr\" deleted\nWaiting for deployment \"ovn-central\" rollout to finish: 0 of 1 updated replicas are available...\ndeployment \"ovn-central\" successfully rolled out\npod \"ovs-ovn-bsnvz\" deleted\npod \"ovs-ovn-m9b98\" deleted\npod \"kube-ovn-controller-8459db5ff4-64c62\" deleted\nWaiting for deployment \"kube-ovn-controller\" rollout to finish: 0 of 1 updated replicas are available...\ndeployment \"kube-ovn-controller\" successfully rolled out\npod \"kube-ovn-cni-2klnh\" deleted\npod \"kube-ovn-cni-t2jz4\" deleted\nWaiting for daemon set \"kube-ovn-cni\" rollout to finish: 0 of 2 updated pods are available...\nWaiting for daemon set \"kube-ovn-cni\" rollout to finish: 1 of 2 updated pods are available...\ndaemon set \"kube-ovn-cni\" successfully rolled out\npod \"kube-ovn-pinger-ln72z\" deleted\npod \"kube-ovn-pinger-w8lrk\" deleted\nWaiting for daemon set \"kube-ovn-pinger\" rollout to finish: 0 of 2 updated pods are available...\nWaiting for daemon set \"kube-ovn-pinger\" rollout to finish: 1 of 2 updated pods are available...\ndaemon set \"kube-ovn-pinger\" successfully rolled out\npod \"kube-ovn-monitor-7fb67d5488-7q6zb\" deleted\nWaiting for deployment \"kube-ovn-monitor\" rollout to finish: 0 of 1 updated replicas are available...\ndeployment \"kube-ovn-monitor\" successfully rolled out\n</code></pre>"},{"location":"en/ops/kubectl-ko/#log","title":"log","text":"<p>Using this command will capture the logs of Kube-OVN, OVN, Openvswitch on all nodes of kube-ovn and some debug information commonly used in linux.</p> <pre><code># kubectl ko log all\nCollecting kube-ovn logging files\nCollecting ovn logging files\nCollecting openvswitch logging files\nCollecting linux dmesg files\nCollecting linux iptables-legacy files\nCollecting linux iptables-nft files\nCollecting linux route files\nCollecting linux link files\nCollecting linux neigh files\nCollecting linux memory files\nCollecting linux top files\nCollecting linux sysctl files\nCollecting linux netstat files\nCollecting linux addr files\nCollecting linux ipset files\nCollecting linux tcp files\nCollected files have been saved in the directory /root/kubectl-ko-log\n</code></pre> <p>The directory is as follows:</p> <pre><code># tree kubectl-ko-log/\nkubectl-ko-log/\n|-- kube-ovn-control-plane\n|   |-- kube-ovn\n|   |   |-- kube-ovn-cni.log\n|   |   |-- kube-ovn-monitor.log\n|   |   `-- kube-ovn-pinger.log\n|   |-- linux\n|   |   |-- addr.log\n|   |   |-- dmesg.log\n|   |   |-- ipset.log\n|   |   |-- iptables-legacy.log\n|   |   |-- iptables-nft.log\n|   |   |-- link.log\n|   |   |-- memory.log\n|   |   |-- neigh.log\n|   |   |-- netstat.log\n|   |   |-- route.log\n|   |   |-- sysctl.log\n|   |   |-- tcp.log\n|   |   `-- top.log\n|   |-- openvswitch\n|   |   |-- ovs-vswitchd.log\n|   |   `-- ovsdb-server.log\n|   `-- ovn\n|       |-- ovn-controller.log\n|       |-- ovn-northd.log\n|       |-- ovsdb-server-nb.log\n|       `-- ovsdb-server-sb.log\n</code></pre>"},{"location":"en/ops/kubectl-ko/#perf-image","title":"perf [image]","text":"<p>This command will test some performance indicators of Kube-OVN as follows:</p> <ol> <li>The performance indicators of the container network;</li> <li>Hostnetwork network performance indicators;</li> <li>Container network multicast packet performance indicators;</li> <li>Time required for OVN-NB, OVN-SB, and OVN-Northd leader deletion recovery. The parameter image is used to specify the image used by the performance test pod. By default, it is <code>kubeovn/test:v1.12.0</code>. This parameter is mainly set for offline scenarios, and the image name may change when the image is pulled to the intranet environment.</li> </ol> <pre><code># kubectl ko perf\n============================== Prepareing Performance Test Resources ===============================\npod/test-client created\npod/test-host-client created\npod/test-server created\npod/test-host-server created\nservice/test-server created\npod/test-client condition met\npod/test-host-client condition met\npod/test-host-server condition met\npod/test-server condition met\n====================================================================================================\n============================ Start Pod Network Unicast Performance Test ============================\nSize            TCP Latency     TCP Bandwidth   UDP Latency     UDP Lost Rate   UDP Bandwidth\n64              82.8 us         97.7 Mbits/sec  67.6 us         (0%)            8.42 Mbits/sec\n128             85.4 us         167 Mbits/sec   67.2 us         (0%)            17.2 Mbits/sec\n512             85.8 us         440 Mbits/sec   68.7 us         (0%)            68.4 Mbits/sec\n1k              85.1 us         567 Mbits/sec   68.7 us         (0%)            134 Mbits/sec\n4k              138 us          826 Mbits/sec   78.1 us         (1.4%)          503 Mbits/sec\n====================================================================================================\n=============================== Start Host Network Performance Test ================================\nSize            TCP Latency     TCP Bandwidth   UDP Latency     UDP Lost Rate   UDP Bandwidth\n64              49.7 us         120 Mbits/sec   37.9 us         (0%)            18.6 Mbits/sec\n128             49.7 us         200 Mbits/sec   38.1 us         (0%)            35.5 Mbits/sec\n512             51.9 us         588 Mbits/sec   38.9 us         (0%)            142 Mbits/sec\n1k              51.7 us         944 Mbits/sec   37.2 us         (0%)            279 Mbits/sec\n4k              74.9 us         1.66 Gbits/sec  39.9 us         (0%)            1.20 Gbits/sec\n====================================================================================================\n============================== Start Service Network Performance Test ==============================\nSize            TCP Latency     TCP Bandwidth   UDP Latency     UDP Lost Rate   UDP Bandwidth\n64              111 us          96.3 Mbits/sec  88.4 us         (0%)            7.59 Mbits/sec\n128             83.7 us         150 Mbits/sec   69.2 us         (0%)            16.9 Mbits/sec\n512             87.4 us         374 Mbits/sec   75.8 us         (0%)            60.9 Mbits/sec\n1k              88.2 us         521 Mbits/sec   73.1 us         (0%)            123 Mbits/sec\n4k              148 us          813 Mbits/sec   77.6 us         (0.0044%)       451 Mbits/sec\n====================================================================================================\n=========================== Start Pod Multicast Network Performance Test ===========================\nSize            UDP Latency     UDP Lost Rate   UDP Bandwidth\n64              0.014 ms        (0.17%)         5.80 Mbits/sec\n128             0.012 ms        (0%)            11.4 Mbits/sec\n512             0.016 ms        (0%)            46.1 Mbits/sec\n1k              0.023 ms        (0.073%)        89.8 Mbits/sec\n4k              0.035 ms        (1.3%)          126 Mbits/sec\n====================================================================================================\n============================= Start Host Multicast Network Performance =============================\nSize            UDP Latency     UDP Lost Rate   UDP Bandwidth\n64              0.007 ms        (0%)            9.95 Mbits/sec\n128             0.005 ms        (0%)            21.8 Mbits/sec\n512             0.008 ms        (0%)            86.8 Mbits/sec\n1k              0.013 ms        (0.045%)        168 Mbits/sec\n4k              0.010 ms        (0.31%)         242 Mbits/sec\n====================================================================================================\n================================== Start Leader Recover Time Test ==================================\nDelete ovn central nb pod\npod \"ovn-central-5cb9c67d75-tlz9w\" deleted\nWaiting for ovn central nb pod running\n=============================== OVN nb Recovery takes 3.305236803 s ================================\nDelete ovn central sb pod\npod \"ovn-central-5cb9c67d75-szx4c\" deleted\nWaiting for ovn central sb pod running\n=============================== OVN sb Recovery takes 3.462698535 s ================================\nDelete ovn central northd pod\npod \"ovn-central-5cb9c67d75-zqmqv\" deleted\nWaiting for ovn central northd pod running\n============================= OVN northd Recovery takes 2.691291403 s ==============================\n====================================================================================================\n================================= Remove Performance Test Resource =================================\nrm -f unicast-test-client.log\nrm -f unicast-test-host-client.log\nrm -f unicast-test-client.log\nkubectl ko nbctl lb-del test-server\nrm -f multicast-test-server.log\nkubectl exec ovs-ovn-gxdrf -n kube-system -- ip maddr del 01:00:5e:00:00:64 dev eth0\nkubectl exec ovs-ovn-h57bf -n kube-system -- ip maddr del 01:00:5e:00:00:64 dev eth0\nrm -f multicast-test-host-server.log\npod \"test-client\" deleted\npod \"test-host-client\" deleted\npod \"test-host-server\" deleted\npod \"test-server\" deleted\nservice \"test-server\" deleted\n====================================================================================================\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/ops/recover-db/","title":"OVN DB Backup and Recovery","text":"<p>This document describes how to perform database backups and how to perform cluster recovery from existing database files in different situations.</p>"},{"location":"en/ops/recover-db/#database-backup","title":"Database Backup","text":"<p>The database files can be backed up for recovery in case of failure. Use the backup command of the kubectl plugin:</p> <pre><code># kubectl ko nb backup\ntar: Removing leading `/' from member names\nbackup ovn-nb db to /root/ovnnb_db.060223191654183154.backup\n\n# kubectl ko sb backup\ntar: Removing leading `/' from member names\nbackup ovn-nb db to /root/ovnsb_db.060223191654183154.backup\n</code></pre>"},{"location":"en/ops/recover-db/#cluster-partial-nodes-failure-recovery","title":"Cluster Partial Nodes Failure Recovery","text":"<p>If some nodes in the cluster are working abnormally due to power failure, file system failure or lack of disk space, but the cluster is still working normally, you can recover it by following the steps below.</p>"},{"location":"en/ops/recover-db/#check-the-logs-to-confirm-status","title":"Check the Logs to Confirm Status","text":"<p>Check the log in <code>/var/log/ovn/ovn-northd.log</code>, if it shows similar error as follows, you can make sue that there is an exception in the database:</p> <pre><code> * ovn-northd is not running\novsdb-server: ovsdb error: error reading record 2739 from OVN_Northbound log: record 2739 advances commit index to 6308 but last log index is 6307\n* Starting ovsdb-nb\n</code></pre>"},{"location":"en/ops/recover-db/#kick-node-from-cluster","title":"Kick Node from Cluster","text":"<p>Select the corresponding database for the operation based on whether the log prompt is <code>OVN_Northbound</code> or <code>OVN_Southbound</code>. The above log prompt is <code>OVN_Northbound</code> then for ovn-nb do the following:</p> <pre><code># kubectl ko nb status\n9182\nName: OVN_Northbound\nCluster ID: e75f (e75fa340-49ed-45ab-990e-26cb865ebc85)\nServer ID: 9182 (9182e8dd-b5b0-4dd8-8518-598cc1e374f3)\nAddress: tcp:[10.0.128.61]:6643\nStatus: cluster member\nRole: leader\nTerm: 1454\nLeader: self\nVote: self\n\nLast Election started 1732603 ms ago, reason: timeout\nLast Election won: 1732587 ms ago\nElection timer: 1000\nLog: [7332, 12512]\nEntries not yet committed: 1\nEntries not yet applied: 1\nConnections: -&gt;f080 &lt;-f080 &lt;-e631 -&gt;e631\nDisconnections: 1\nServers:\n    f080 (f080 at tcp:[10.0.129.139]:6643) next_index=12512 match_index=12510 last msg 63 ms ago\n    9182 (9182 at tcp:[10.0.128.61]:6643) (self) next_index=10394 match_index=12510\ne631 (e631 at tcp:[10.0.131.173]:6643) next_index=12512 match_index=0\n</code></pre> <p>Kick abnormal nodes from the cluster:</p> <pre><code>kubectl ko nb kick e631\n</code></pre> <p>Log in to the abnormal node and delete the database file:</p> <pre><code>mv /etc/origin/ovn/ovnnb_db.db /tmp\n</code></pre> <p>Delete the <code>ovn-central</code> pod of the corresponding node and wait for the cluster to recover\uff1a</p> <pre><code>kubectl delete pod -n kube-system ovn-central-xxxx\n</code></pre>"},{"location":"en/ops/recover-db/#recover-when-total-cluster-failed","title":"Recover when Total Cluster Failed","text":"<p>If the majority of the cluster nodes are broken and the leader cannot be elected, please refer to the following steps to recover.</p>"},{"location":"en/ops/recover-db/#stop-ovn-central","title":"Stop ovn-central","text":"<p>Record the current replicas of <code>ovn-central</code> and stop <code>ovn-central</code> to avoid new database changes that affect recovery:</p> <pre><code>kubectl scale deployment -n kube-system ovn-central --replicas=0\n</code></pre>"},{"location":"en/ops/recover-db/#select-a-backup","title":"Select a Backup","text":"<p>As most of the nodes are damaged, the cluster needs to be rebuilt by recovering from one of the database files. If you have previously backed up the database you can use the previous backup file to restore it. If not you can use the following steps to generate a backup from an existing file.</p> <p>Since the database file in the default folder is a cluster format database file containing information about the current cluster, you can't rebuild the database directly with this file, you need to use <code>ovsdb-tool cluster-to-standalone</code> to convert the format.</p> <p>Select the first node in the <code>ovn-central</code> environment variable <code>NODE_IPS</code> to restore the database files. If the database file of the first node is corrupted, copy the file from the other machine <code>/etc/origin/ovn</code> to the first machine. Run the following command to generate a database file backup.</p> <pre><code>docker run -it -v /etc/origin/ovn:/etc/ovn kubeovn/kube-ovn:v1.13.0 bash\ncd /etc/ovn/\novsdb-tool cluster-to-standalone ovnnb_db_standalone.db ovnnb_db.db\novsdb-tool cluster-to-standalone ovnsb_db_standalone.db ovnsb_db.db\n</code></pre>"},{"location":"en/ops/recover-db/#delete-the-database-files-on-all-ovn-central-nodes","title":"Delete the Database Files on All ovn-central Nodes","text":"<p>In order to avoid rebuilding the cluster with the wrong data, the existing database files need to be cleaned up:</p> <pre><code>mv /etc/origin/ovn/ovnnb_db.db /tmp\nmv /etc/origin/ovn/ovnsb_db.db /tmp\n</code></pre>"},{"location":"en/ops/recover-db/#recovering-database-cluster","title":"Recovering Database Cluster","text":"<p>Rename the backup databases to <code>ovnnb_db.db</code> and <code>ovnsb_db.db</code> respectively, and copy them to the <code>/etc/origin/ovn/</code> directory of the first machine in the <code>ovn-central</code> environment variable <code>NODE_IPS</code>\uff1a</p> <pre><code>mv /etc/origin/ovn/ovnnb_db_standalone.db /etc/origin/ovn/ovnnb_db.db\nmv /etc/origin/ovn/ovnsb_db_standalone.db /etc/origin/ovn/ovnsb_db.db\n</code></pre> <p>Restore the number of replicas of <code>ovn-central</code>\uff1a</p> <pre><code>kubectl scale deployment -n kube-system ovn-central --replicas=3\nkubectl rollout status deployment/ovn-central -n kube-system\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/reference/architecture/","title":"Architecture","text":"<p>This document describes the general architecture of Kube-OVN, the functionality of each component and how they interact with each other.</p> <p>Overall, Kube-OVN serves as a bridge between Kubernetes and OVN, combining proven SDN with Cloud Native. This means that Kube-OVN not only implements network specifications under Kubernetes, such as CNI, Service and Networkpolicy, but also brings a large number of SDN domain capabilities to cloud-native, such as logical switches, logical routers, VPCs, gateways, QoS, ACLs and traffic mirroring.</p> <p>Kube-OVN also maintains a good openness to integrate with many technology solutions, such as Cilium, Submariner, Prometheus, KubeVirt, etc.</p>"},{"location":"en/reference/architecture/#component-introduction","title":"Component Introduction","text":"<p>The components of Kube-OVN can be broadly divided into three categories.</p> <ul> <li>Upstream OVN/OVS components.</li> <li>Core Controller and Agent.</li> <li>Monitoring, operation and maintenance tools and extension components.</li> </ul> <p></p>"},{"location":"en/reference/architecture/#upstream-ovnovs-components","title":"Upstream OVN/OVS Components","text":"<p>This type of component comes from the OVN/OVS community with specific modifications for Kube-OVN usage scenarios. OVN/OVS itself is a mature SDN system for managing virtual machines and containers, and we strongly recommend that users interested in the Kube-OVN implementation read ovn-architecture(7) first to understand what OVN is and how to integrate with it. Kube-OVN uses the northbound interface of OVN to create and coordinate virtual networks and map the network concepts into Kubernetes.</p> <p>All OVN/OVS-related components have been packaged into images and are ready to run in Kubernetes.</p>"},{"location":"en/reference/architecture/#ovn-central","title":"ovn-central","text":"<p>The <code>ovn-central</code> Deployment runs the control plane components of OVN, including <code>ovn-nb</code>, <code>ovn-sb</code>, and <code>ovn-northd</code>.</p> <ul> <li><code>ovn-nb</code>: Saves the virtual network configuration and provides an API for virtual network management. <code>kube-ovn-controller</code> will mainly interact with <code>ovn-nb</code> to configure the virtual network.</li> <li><code>ovn-sb</code>: Holds the logical flow table generated from the logical network of <code>ovn-nb</code>, as well as the actual physical network state of each node.</li> <li><code>ovn-northd</code>: translates the virtual network of <code>ovn-nb</code> into a logical flow table in <code>ovn-sb</code>.</li> </ul> <p>Multiple instances of <code>ovn-central</code> will synchronize data via the Raft protocol to ensure high availability.</p>"},{"location":"en/reference/architecture/#ovs-ovn","title":"ovs-ovn","text":"<p><code>ovs-ovn</code> runs as a DaemonSet on each node, with <code>openvswitch</code>, <code>ovsdb</code>, and <code>ovn-controller</code> running inside the Pod. These components act as agents for <code>ovn-central</code> to translate logical flow tables into real network configurations.</p>"},{"location":"en/reference/architecture/#core-controller-and-agent","title":"Core Controller and Agent","text":"<p>This part is the core component of Kube-OVN, serving as a bridge between OVN and Kubernetes, bridging the two systems and translating network concepts between them. Most of the core functions are implemented in these components.</p>"},{"location":"en/reference/architecture/#kube-ovn-controller","title":"kube-ovn-controller","text":"<p>This component performs the translation of all resources within Kubernetes to OVN resources and acts as the control plane for the entire Kube-OVN system. The <code>kube-ovn-controller</code> listens for events on all resources related to network functionality and updates the logical network within the OVN based on resource changes. The main resources listened including:</p> <p>Pod\uff0cService\uff0cEndpoint\uff0cNode\uff0cNetworkPolicy\uff0cVPC\uff0cSubnet\uff0cVlan\uff0cProviderNetwork\u3002</p> <p>Taking the Pod event as an example, <code>kube-ovn-controller</code> listens to the Pod creation event, allocates the address via the built-in in-memory IPAM function, and calls <code>ovn-central</code> to create logical ports, static routes and possible ACL rules. Next, <code>kube-ovn-controller</code> writes the assigned address and subnet information such as CIDR, gateway, route, etc. to the annotation of the Pod. This annotation is then read by <code>kube-ovn-cni</code> and used to configure the local network.</p>"},{"location":"en/reference/architecture/#kube-ovn-cni","title":"kube-ovn-cni","text":"<p>This component runs on each node as a DaemonSet, implements the CNI interface, and operates the local OVS to configure the local network.</p> <p>This DaemonSet copies the <code>kube-ovn</code> binary to each machine as a tool for interaction between <code>kubelet</code> and <code>kube-ovn-cni</code>. This binary sends the corresponding CNI request to <code>kube-ovn-cni</code> for further operation. The binary will be copied to the <code>/opt/cni/bin</code> directory by default.</p> <p><code>kube-ovn-cni</code> will configure the specific network to perform the appropriate traffic operations, and the main tasks including:</p> <ol> <li>Config <code>ovn-controller</code> and <code>vswitchd</code>.</li> <li>Handle CNI Add/Del requests:<ol> <li>Create or delete veth pair and bind or unbind to OVS ports.</li> <li>Configure OVS ports</li> <li>Update host iptables/ipset/route rules.</li> </ol> </li> <li>Dynamically update the network QoS.</li> <li>Create and configure the <code>ovn0</code> NIC to connect the container network and the host network.</li> <li>Configure the host NIC to implement Vlan/Underlay/EIP.</li> <li>Dynamically config inter-cluster gateways.</li> </ol>"},{"location":"en/reference/architecture/#monitoring-operation-and-maintenance-tools-and-extension-components","title":"Monitoring, Operation and Maintenance Tools and Extension Components","text":"<p>These components provide monitoring, diagnostics, operations tools, and external interface to extend the core network capabilities of Kube-OVN and simplify daily operations and maintenance.</p>"},{"location":"en/reference/architecture/#kube-ovn-speaker","title":"kube-ovn-speaker","text":"<p>This component is a DaemonSet running on a specific labeled nodes that publish routes to the external, allowing external access to the container directly through the Pod IP.</p> <p>For more information on how to use it, please refer to BGP Support.</p>"},{"location":"en/reference/architecture/#kube-ovn-pinger","title":"kube-ovn-pinger","text":"<p>This component is a DaemonSet running on each node to collect OVS status information, node network quality, network latency, etc. The monitoring metrics collected can be found in Metrics.</p>"},{"location":"en/reference/architecture/#kube-ovn-monitor","title":"kube-ovn-monitor","text":"<p>This component collects OVN status information and the monitoring metrics, all metrics can be found in Metrics.</p>"},{"location":"en/reference/architecture/#kubectl-ko","title":"kubectl-ko","text":"<p>This component is a kubectl plugin, which can quickly run common operations, for more usage, please refer to [kubectl plugin].(../ops/kubectl-ko.en.md)\u3002</p> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/reference/dev-env/","title":"Development Setup","text":""},{"location":"en/reference/dev-env/#environmental-preparation","title":"Environmental Preparation","text":"<p>Kube-OVN uses Golang 1.20 to develop and Go Modules to manage dependency, please check env <code>GO111MODULE=\"on\"</code>\u3002</p> <p>gosec is used to scan for code security related issues and requires to be installed in the development environment:</p> <pre><code>go get github.com/securego/gosec/v2/cmd/gosec\n</code></pre> <p>To reduce the size of the final generated image, Kube-OVN uses some of the Docker buildx experimental features, please update Docker to the latest version and enable buildx:</p> <pre><code>docker buildx create --use\n</code></pre>"},{"location":"en/reference/dev-env/#build-image","title":"Build Image","text":"<p>Use the following command to download the code and generate the image required to run Kube-OVN:</p> <pre><code>git clone https://github.com/kubeovn/kube-ovn.git\ncd kube-ovn\nmake release\n</code></pre> <p>To build an image to run in an ARM environment, run the following command:</p> <pre><code>make release-arm\n</code></pre>"},{"location":"en/reference/dev-env/#building-the-base-image","title":"Building the Base Image","text":"<p>If you need to change the operating system version, dependencies, OVS/OVN code, etc., you need to rebuild the base image.</p> <p>The Dockerfile used for the base image is <code>dist/images/Dockerfile.base</code>.</p> <p>Build instructions:</p> <pre><code># build x86 base image\nmake base-amd64\n\n# build arm base image\nmake base-arm64\n</code></pre>"},{"location":"en/reference/dev-env/#run-e2e","title":"Run E2E","text":"<p>Kube-OVN uses KIND to build local Kubernetes cluster, j2cli to render templates, and Ginkgo to run test cases. Please refer to the relevant documentation for dependency installation.</p> <p>Run E2E locally:</p> <pre><code>make kind-init\nmake kind-install\nmake e2e\n</code></pre> <p>To run the Underlay E2E test, run the following commands:</p> <pre><code>make kind-init\nmake kind-install-underlay\nmake e2e-underlay-single-nic\n</code></pre> <p>To run the ovn vpc nat gw eip, fip, snat, dnat E2E test, run the following commands:</p> <pre><code>make kind-init\nmake kind-install\nmake ovn-vpc-nat-gw-conformance-e2e\n</code></pre> <p>To run the iptables vpc nat gw eip, fip, snat, dnat E2E test, run the following commands:</p> <pre><code>make kind-init\nmake kind-install\nmake kind-install-vpc-nat-gw\nmake iptables-vpc-nat-gw-conformance-e2e\n</code></pre> <p>To run the loadbalancer service E2E test, run the following commands:</p> <pre><code>make kind-init\nmake kind-install\nmake kind-install-lb-svc\nmake kube-ovn-lb-svc-conformance-e2e\n</code></pre> <p>To clean, run the following commands:</p> <pre><code>make kind-clean\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/reference/document-convention/","title":"Document Specification","text":"<p>In order to ensure a consistent document style, please follow the following style guidelines when submitting documents.</p>"},{"location":"en/reference/document-convention/#punctuation","title":"Punctuation","text":"<p>All punctuation in the text content in Chinese documents should use Chinese format punctuation, and all text content in English documents should use English punctuation.</p> BadGood   Here is a one-click installation script that can help you quickly install a highly available, production-ready container network.    Here is a one-click installation script that can help you quickly install a highly available, production-ready container network.   <p>English numbers and Chinese characters should be separated by spaces.</p> BadGood   Kube-OVN provides a one-click installation script to install version 1.10 of Kube-OVN.    Kube-OVN provides a one-click installation script to install version 1.10 of Kube-OVN.   <p>Example content should start with <code>:</code>, other sentences should end with <code>.</code> End.</p> BadGood   Please confirm that the environment configuration is correct before installation  Download the installation script using the command below.  <pre><code>wget 127.0.0.1\n</code></pre>   Please confirm that the environment configuration is correct before installation.  Download the installation script using the following command:  <pre><code>wget 127.0.0.1\n</code></pre>"},{"location":"en/reference/document-convention/#code-block","title":"Code Block","text":"<p>yaml code blocks need to be identified as yaml.</p> BadGood <pre><code>````\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n     name: attach-subnet\n````\n</code></pre> <pre><code>````yaml\napiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\n     name: attach-subnet\n````\n</code></pre> <p>Command-line manipulation example code blocks need to be identified as bash.</p> BadGood <pre><code>````\nwget 127.0.0.1\n````\n</code></pre> <pre><code>````bash\nwget 127.0.0.1\n````\n</code></pre> <p>If the command line operation example contains output content, the executed command needs to start with <code>#</code> to distinguish input from output.</p> BadGood <pre><code>oilbeater@macdeMac-3 ~ ping 114.114.114.114 -c 3\nPING 114.114.114.114 (114.114.114.114): 56 data bytes\n64 bytes from 114.114.114.114: icmp_seq=0 ttl=83 time=10.429 ms\n64 bytes from 114.114.114.114: icmp_seq=1 ttl=79 time=11.360 ms\n64 bytes from 114.114.114.114: icmp_seq=2 ttl=76 time=10.794 ms\n\n--- 114.114.114.114 ping statistics ---\n3 packets transmitted, 3 packets received, 0.0% packet loss\nround-trip min/avg/max/stddev = 10.429/10.861/11.360/0.383 ms\n</code></pre> <pre><code># ping 114.114.114.114 -c 3\nPING 114.114.114.114 (114.114.114.114): 56 data bytes\n64 bytes from 114.114.114.114: icmp_seq=0 ttl=83 time=10.429 ms\n64 bytes from 114.114.114.114: icmp_seq=1 ttl=79 time=11.360 ms\n64 bytes from 114.114.114.114: icmp_seq=2 ttl=76 time=10.794 ms\n\n--- 114.114.114.114 ping statistics ---\n3 packets transmitted, 3 packets received, 0.0% packet loss\nround-trip min/avg/max/stddev = 10.429/10.861/11.360/0.383 ms\n</code></pre> <p>If the command line operation example only contains execution commands and no output results, multiple commands do not need to start with <code>#</code>.</p> BadGood <pre><code># mv /etc/origin/ovn/ovnnb_db.db /tmp\n# mv /etc/origin/ovn/ovnsb_db.db /tmp\n</code></pre> <pre><code>mv /etc/origin/ovn/ovnnb_db.db /tmp\nmv /etc/origin/ovn/ovnsb_db.db /tmp\n</code></pre>"},{"location":"en/reference/document-convention/#link","title":"Link","text":"<p>Links in the site use the corresponding <code>md</code> file path.</p> BadGood <pre><code>Please refer to [Preparation](http://kubeovn.github.io/prepare) before installation.\n</code></pre> <pre><code>Please refer to [Preparation](./prepare.md) before installation.\n</code></pre> BadGood <pre><code>If you have any questions, please refer to [Kubernetes Documentation](http://kubernetes.io).\n</code></pre> <pre><code>If you have any questions, please refer to [Kubernetes Documentation](http://kubernetes.io){: target=\"_blank\" }.\n</code></pre>"},{"location":"en/reference/document-convention/#empty-line","title":"Empty Line","text":"<p>Different logical blocks, such as title and text, text and code, text and number need to be separated by blank lines.</p> BadGood <pre><code>Download the script below to install it:\n```bash\nwget 127.0.0.1\n```\n</code></pre> <pre><code>Download the script below to install it:\n\n```bash\nwget 127.0.0.1\n```\n</code></pre> <p>Separate logical blocks with only one blank line.</p> BadGood <pre><code>Download the script below to install it:\n\n\n\n```bash\nwget 127.0.0.1\n```\n</code></pre> <pre><code>Download the script below to install it:\n\n```bash\nwget 127.0.0.1\n```\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/reference/feature-stage/","title":"Feature Stage","text":"<p>In Kube-OVN, feature stage is classified into Alpha, Beta and GA, based on the degree of feature usage, documentation and test coverage.</p>"},{"location":"en/reference/feature-stage/#definition-of-stage","title":"Definition of Stage","text":"<p>For Alpha stage functions:</p> <ul> <li>The feature is not fully documented and well tested.</li> <li>This feature may change or even be removed in the future.</li> <li>This feature API is not guaranteed to be stable and may be removed.</li> <li>Community provides low priority support for this feature and long-term support cannot be guaranteed.</li> <li>Since feature stability and long-term support cannot be guaranteed, it can be tested and verified, but is not recommended for production use.</li> </ul> <p>For Beta stage functions:</p> <ul> <li>This feature is partially documented and tested, but complete coverage is not guaranteed.</li> <li>This feature may change in the future and the upgrade may affect the network, but it will not be removed as a whole.</li> <li>This feature API may change in the future and the fields may be adjusted, but not removed as a whole.</li> <li>This feature will be supported by the community in the long term.</li> <li>It can be used on non-critical services as the functionality will be supported for a long time, but it is not recommended for critical production service as there is a possibility of changes in functionality and APIs that may break the network.</li> </ul> <p>For GA stage functions:</p> <ul> <li>The feature has full documentation and test coverage.</li> <li>The feature will remain stable and upgrades will be guaranteed to be smooth.</li> <li>This feature API is not subject to disruptive changes.</li> <li>This feature will be supported with high priority by the community and long-term support will be guaranteed.</li> </ul>"},{"location":"en/reference/feature-stage/#feature-stage-list","title":"Feature Stage List","text":"<p>This list records the feature stages from the 1.8 release.</p> Feature Default Stage Since Until Namespaced Subnet true GA 1.8 Distributed Gateway true GA 1.8 Active-backup Centralized Gateway true GA 1.8 ECMP Centralized Gateway false Beta 1.8 Subnet ACL true Alpha 1.9 Subnet Isolation (Will be replaced by ACL later) true Beta 1.8 Underlay Subnet true GA 1.8 Multiple Pod Interface true Beta 1.8 Subnet DHCP false Alpha 1.10 Subnet with External Gateway false Alpha 1.8 Cluster Inter-Connection with OVN-IC false Beta 1.8 Cluster Inter-Connection with Submariner false Alpha 1.9 VIP Reservation true Alpha 1.10 Create Custom VPC true Beta 1.8 Custom VPC Floating IP/SNAT/DNAT true Alpha 1.10 Custom VPC Static Route true Alpha 1.10 Custom VPC Policy Route true Alpha 1.10 Custom VPC Security Group true Alpha 1.10 Container Bandwidth QoS true GA 1.8 linux-netem QoS true Alpha 1.9 Prometheus Integration false GA 1.8 Grafana Integration false GA 1.8 IPv4/v6 DualStack false GA 1.8 Default VPC EIP/SNAT false Beta 1.8 Traffic Mirroring false GA 1.8 NetworkPolicy true Beta 1.8 Webhook false Alpha 1.10 Performance Tunning false Beta 1.8 Interconnection with Routes in Overlay Mode false Alpha 1.8 BGP Support false Alpha 1.9 Cilium Integration false Alpha 1.10 Custom VPC Peering false Alpha 1.10 Mellanox Offload false Alpha 1.8 Corigine Offload false Alpha 1.10 Windows Support false Alpha 1.10 DPDK Support false Alpha 1.10 OpenStack Integration false Alpha 1.9 Single Pod Fixed IP/Mac true GA 1.8 Workload with Fixed IP true GA 1.8 StatefulSet with Fixed IP true GA 1.8 VM with Fixed IP false Beta 1.9 Load Balancer Type Service in Default VPC false Alpha 1.11 Load Balance in Custom VPC false Alpha 1.11 DNS in Custom VPC false Alpha 1.11 Underlay and Overlay Interconnection false Alpha 1.11 <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/reference/iptables-rules/","title":"Iptables Rules","text":"<p>Kube-OVN uses <code>ipset</code> and <code>iptables</code> to implement gateway NAT functionality in the default VPC overlay Subnets.</p> <p>The ipset used is shown in the following table:</p> Name\uff08IPv4/IPv6\uff09 Type Usage ovn40services/ovn60services hash:net Service CIDR ovn40subnets/ovn60subnets hash:net Overlay Subnet CIDR and NodeLocal DNS IP address ovn40subnets-nat/ovn60subnets-nat hash:net Overlay Subnet CIDRs that enable <code>NatOutgoing</code> ovn40subnets-distributed-gw/ovn60subnets-distributed-gw hash:net Overlay Subnet CIDRs that use distributed gateway ovn40other-node/ovn60other-node hash:net Internal IP addresses for other Nodes ovn40local-pod-ip-nat/ovn60local-pod-ip-nat hash:ip Deprecated ovn40subnets-nat-policy hash:net All subnet cidrs configured with natOutgoingPolicyRules ovn40natpr-418e79269dc5-dst hash:net The dstIPs corresponding to the rule in natOutgoingPolicyRules ovn40natpr-418e79269dc5-src hash:net The srcIPs corresponding to the rule in natOutgoingPolicyRules <p>The iptables rules (IPv4) used are shown in the following table:</p> Table Chain Rule Usage Note filter INPUT -m set --match-set ovn40services src -j ACCEPT Allow k8s service and pod traffic to pass through -- filter INPUT -m set --match-set ovn40services dst -j ACCEPT Allow k8s service and pod traffic to pass through -- filter INPUT -m set --match-set ovn40subnets src -j ACCEPT Allow k8s service and pod traffic to pass through -- filter INPUT -m set --match-set ovn40subnets dst -j ACCEPT Allow k8s service and pod traffic to pass through -- filter FORWARD -m set --match-set ovn40services src -j ACCEPT Allow k8s service and pod traffic to pass through -- filter FORWARD -m set --match-set ovn40services dst -j ACCEPT Allow k8s service and pod traffic to pass through -- filter FORWARD -m set --match-set ovn40subnets src -j ACCEPT Allow k8s service and pod traffic to pass through -- filter FORWARD -m set --match-set ovn40subnets dst -j ACCEPT Allow k8s service and pod traffic to pass through -- filter FORWARD -s 10.16.0.0/16 -m comment --comment \"ovn-subnet-gateway,ovn-default\" Used to count packets from the subnet to the external network \"10.16.0.0/16\" is the cidr of the subnet, the \"ovn-subnet-gateway\" before the \",\" in comment is used to identify the iptables rule used to count the subnet inbound and outbound gateway packets, and the \"ovn-default\" after the \",\" is the name of the subnet filter FORWARD -d 10.16.0.0/16 -m comment --comment \"ovn-subnet-gateway,ovn-default\" Used to count packets from the external network accessing the subnet \"10.16.0.0/16\" is the cidr of the subnet, the \"ovn-subnet-gateway\" before the \",\" in comment is used to identify the iptables rule used to count the subnet inbound and outbound gateway packets, and the \"ovn-default\" after the \",\" is the name of the subnet filter OUTPUT -p udp -m udp --dport 6081 -j MARK --set-xmark 0x0 Clear traffic tag to prevent SNAT UDP: bad checksum on VXLAN interface nat PREROUTING -m comment --comment \"kube-ovn prerouting rules\" -j OVN-PREROUTING Enter OVN-PREROUTING chain processing -- nat POSTROUTING -m comment --comment \"kube-ovn postrouting rules\" -j OVN-POSTROUTING Enter OVN-POSTROUTING chain processing -- nat OVN-PREROUTING -i ovn0 -m set --match-set ovn40subnets src -m set --match-set ovn40services dst -j MARK --set-xmark 0x4000/0x4000 Adding masquerade tags to Pod access service traffic Used when the built-in LB is turned off nat OVN-PREROUTING -p tcp -m addrtype --dst-type LOCAL -m set --match-set KUBE-NODE-PORT-LOCAL-TCP dst -j MARK --set-xmark 0x80000/0x80000 Add specific tags to ExternalTrafficPolicy for Local's Service traffic (TCP) Only used when kube-proxy is using ipvs mode nat OVN-PREROUTING -p udp -m addrtype --dst-type LOCAL -m set --match-set KUBE-NODE-PORT-LOCAL-UDP dst -j MARK --set-xmark 0x80000/0x80000 Add specific tags to ExternalTrafficPolicy for Local's Service traffic (UDP) Only used when kube-proxy is using ipvs mode nat OVN-POSTROUTING -m set --match-set ovn40services src -m set --match-set ovn40subnets dst -m mark --mark 0x4000/0x4000 -j SNAT --to-source  Use node IP as the source address for access from node to overlay Pods via service IP\u3002 Works only when kube-proxy is using ipvs mode nat OVN-POSTROUTING -m mark --mark 0x4000/0x4000 -j MASQUERADE Perform SNAT for specific tagged traffic -- nat OVN-POSTROUTING -m set --match-set ovn40subnets src -m set --match-set ovn40subnets dst -j MASQUERADE Perform SNAT for Service traffic between Pods passing through the node -- nat OVN-POSTROUTING -m mark --mark 0x80000/0x80000 -m set --match-set ovn40subnets-distributed-gw dst -j RETURN For Service traffic where ExternalTrafficPolicy is Local, if the Endpoint uses a distributed gateway, SNAT is not required. -- nat OVN-POSTROUTING -m mark --mark 0x80000/0x80000 -j MASQUERADE For Service traffic where ExternalTrafficPolicy is Local, if the Endpoint uses a centralized gateway, SNAT is required. -- nat OVN-POSTROUTING -p tcp -m tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN No SNAT is performed when the Pod IP is exposed to the outside world -- nat OVN-POSTROUTING -s 10.16.0.0/16 -m set ! --match-set ovn40subnets dst -j SNAT --to-source 192.168.0.101 When the Pod accesses the network outside the cluster, if the subnet is NatOutgoing and a centralized gateway with the specified IP is used, perform SNAT 10.16.0.0/16 is the Subnet CIDR\uff0c192.168.0.101 is the specified IP of gateway node nat OVN-POSTROUTING -m set --match-set ovn40subnets-nat src -m set ! --match-set ovn40subnets dst -j MASQUERADE When the Pod accesses the network outside the cluster, if NatOutgoing is enabled on the subnet, perform SNAT -- nat OVN-POSTROUTING -m set --match-set ovn40subnets-nat-policy src -m set ! --match-set ovn40subnets dst -j OVN-NAT-POLICY When Pod accesses the network outside the cluster, if natOutgoingPolicyRules is enabled on the subnet, the packet with the specified policy will perform SNAT ovn40subnets-nat-policy is all subnet segments configured with natOutgoingPolicyRules nat OVN-POSTROUTING -m mark --mark 0x90001/0x90001 -j MASQUERADE --random-fully When Pod accesses the network outside the cluster, if natOutgoingPolicyRules is enabled on the subnet, the packet with the specified policy will perform SNAT After coming out of OVN-NAT-POLICY, if it is tagged with 0x90001/0x90001, it will do SNAT nat OVN-POSTROUTING -m mark --mark 0x90002/0x90002 -j RETURN When Pod accesses the network outside the cluster, if natOutgoingPolicyRules is enabled on the subnet, the packet with the specified policy will perform SNAT After coming out of OVN-NAT-POLICY, if it is tagged with 0x90002/0x90002, it will not do SNAT nat OVN-NAT-POLICY -s 10.0.11.0/24 -m comment --comment natPolicySubnet-net1 -j OVN-NAT-PSUBNET-aa98851157c5 When Pod accesses the network outside the cluster, if natOutgoingPolicyRules is enabled on the subnet, the packet with the specified policy will perform SNAT 10.0.11.0/24 represents the CIDR of the subnet net1, and the rules under the OVN-NAT-PSUBNET-aa98851157c5 chain correspond to the natOutgoingPolicyRules configuration of this subnet nat OVN-NAT-PSUBNET-xxxxxxxxxxxx -m set --match-set ovn40natpr-418e79269dc5-src src -m set --match-set ovn40natpr-418e79269dc5-dst dst -j MARK --set-xmark 0x90002/0x90002 When Pod accesses the network outside the cluster, if natOutgoingPolicyRules is enabled on the subnet, the packet with the specified policy will perform SNAT 418e79269dc5 indicates the ID of a rule in natOutgoingPolicyRules, which can be viewed through status.natOutgoingPolicyRules[index].RuleID, indicating that srcIPs meets ovn40natpr-418e79269dc5-src, and dstIPS meets ovn40natpr-418e79269dc5- dst will be marked with tag 0x90002 mangle OVN-OUTPUT -d 10.241.39.2/32 -p tcp -m tcp --dport 80 -j MARK --set-xmark 0x90003/0x90003 Introduce kubelet's detection traffic to tproxy with a specific mark mangle OVN-PREROUTING -d 10.241.39.2/32 -p tcp -m tcp --dport 80 -j TPROXY --on-port 8102 --on-ip 172.18.0.3 --tproxy-mark 0x90004/0x90004 Introduce kubelet's detection traffic to tproxy with a specific mark <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/reference/kube-ovn-api/","title":"Kube-OVN API Reference","text":"<p>Based on Kube-OVN v1.12.0, we have compiled a list of CRD resources supported by Kube-OVN, listing the types and meanings of each field of CRD definition for reference.</p>"},{"location":"en/reference/kube-ovn-api/#generic-condition-definition","title":"Generic Condition Definition","text":"Property Name Type Description type String Type of status status String The value of status, in the range of <code>True</code>, <code>False</code> or <code>Unknown</code> reason String The reason for the status change message String The specific message of the status change lastUpdateTime Time The last time the status was updated lastTransitionTime Time Time of last status type change <p>In each CRD definition, the Condition field in Status follows the above format, so we explain it in advance.</p>"},{"location":"en/reference/kube-ovn-api/#subnet-definition","title":"Subnet Definition","text":""},{"location":"en/reference/kube-ovn-api/#subnet","title":"Subnet","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources have this value as kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource will have the value <code>Subnet</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec SubnetSpec Subnet specific configuration information status SubnetStatus Subnet status information"},{"location":"en/reference/kube-ovn-api/#subnetspec","title":"SubnetSpec","text":"Property Name Type Description default Bool Whether this subnet is the default subnet vpc String The vpc which the subnet belongs to, default is ovn-cluster protocol String IP protocol, the value is in the range of <code>IPv4</code>, <code>IPv6</code> or <code>Dual</code> namespaces []String The list of namespaces bound to this subnet cidrBlock String The range of the subnet, e.g. 10.16.0.0/16 gateway String The gateway address of the subnet, the default value is the first available address under the CIDRBlock of the subnet excludeIps []String The range of addresses under this subnet that will not be automatically assigned provider String Default value is <code>ovn</code>. In the case of multiple NICs, the value is <code>&lt;name&gt;.&lt;namespace&gt;</code> of the NetworkAttachmentDefinition, Kube-OVN will use this information to find the corresponding subnet resource gatewayType String The gateway type in overlay mode, either <code>distributed</code> or <code>centralized</code> gatewayNode String The gateway node when the gateway mode is centralized, node names can be comma-separated natOutgoing Bool Whether the outgoing traffic is NAT externalEgressGateway String The address of the external gateway. This parameter and the natOutgoing parameter cannot be set at the same time policyRoutingPriority Uint32 Policy route priority. Used to control the forwarding of traffic to the external gateway address after the subnet gateway policyRoutingTableID Uint32 The TableID of the local policy routing table, should be different for each subnet to avoid conflicts private Bool Whether the subnet is a private subnet, which denies access to addresses inside the subnet if the subnet is private allowSubnets []String If the subnet is a private subnet, the set of addresses that are allowed to access the subnet vlan String The name of vlan to which the subnet is bound vips []String The virtual-ip parameter information for virtual type lsp on the subnet logicalGateway Bool Whether to enable logical gateway disableGatewayCheck Bool Whether to skip the gateway connectivity check when creating a pod disableInterConnection Bool Whether to enable subnet interconnection across clusters enableDHCP Bool Whether to configure dhcp configuration options for lsps belong this subnet dhcpV4Options String The DHCP_Options record associated with lsp dhcpv4_options on the subnet dhcpV6Options String The DHCP_Options record associated with lsp dhcpv6_options on the subnet enableIPv6RA Bool Whether to configure the ipv6_ra_configs parameter for the lrp port of the router connected to the subnet ipv6RAConfigs String The ipv6_ra_configs parameter configuration for the lrp port of the router connected to the subnet acls []Acl The acls record associated with the logical-switch of the subnet u2oInterconnection Bool Whether to enable interconnection mode for Overlay/Underlay enableLb *Bool Whether the logical-switch of the subnet is associated with load-balancer records enableEcmp Bool Centralized subnet, whether to enable ECMP routing"},{"location":"en/reference/kube-ovn-api/#acl","title":"Acl","text":"Property Name Type Description direction String Restrict the direction of acl, which value is <code>from-lport</code> or <code>to-lport</code> priority Int Acl priority, in the range 0 to 32767 match String Acl rule match expression action String The action of the rule, which value is in the range of <code>allow-related</code>, <code>allow-stateless</code>, <code>allow</code>, <code>drop</code>, <code>reject</code>"},{"location":"en/reference/kube-ovn-api/#subnetstatus","title":"SubnetStatus","text":"Property Name Type Description conditions []SubnetCondition Subnet status change information, refer to the beginning of the document for the definition of Condition v4AvailableIPs Float64 Number of available IPv4 IPs v4availableIPrange String The available range of IPv4 addresses on the subnet v4UsingIPs Float64 Number of used IPv4 IPs v4usingIPrange String Used IPv4 address ranges on the subnet v6AvailableIPs Float64 Number of available IPv6 IPs v6availableIPrange String The available range of IPv6 addresses on the subnet v6UsingIPs Float64 Number of used IPv6 IPs v6usingIPrange String Used IPv6 address ranges on the subnet sctivateGateway String The currently working gateway node in centralized subnet of master-backup mode dhcpV4OptionsUUID String The DHCP_Options record identifier associated with the lsp dhcpv4_options on the subnet dhcpV6OptionsUUID String The DHCP_Options record identifier associated with the lsp dhcpv6_options on the subnet u2oInterconnectionIP String The IP address used for interconnection when Overlay/Underlay interconnection mode is enabled"},{"location":"en/reference/kube-ovn-api/#ip-definition","title":"IP Definition","text":""},{"location":"en/reference/kube-ovn-api/#ip","title":"IP","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources are kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource have the value <code>IP</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec IPSpec IP specific configuration information"},{"location":"en/reference/kube-ovn-api/#ipsepc","title":"IPSepc","text":"Property Name Type Description podName String Pod name which assigned with this IP namespace String The name of the namespace where the pod is bound subnet String The subnet which the ip belongs to attachSubnets []String The name of the other subnets attached to this primary IP (field deprecated) nodeName String The name of the node where the pod is bound ipAddress String IP address, in <code>v4IP,v6IP</code> format for dual-stack cases v4IPAddress String IPv4 IP address v6IPAddress String IPv6 IP address attachIPs []String Other IP addresses attached to this primary IP (field is deprecated) macAddress String The Mac address of the bound pod attachMacs []String Other Mac addresses attached to this primary IP (field deprecated) containerID String The Container ID corresponding to the bound pod podType String Special workload pod, can be <code>StatefulSet</code>, <code>VirtualMachine</code> or empty"},{"location":"en/reference/kube-ovn-api/#underlay-configuration","title":"Underlay configuration","text":""},{"location":"en/reference/kube-ovn-api/#vlan","title":"Vlan","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all instances of this resource will be kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource will have the value <code>Vlan</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec VlanSpec Vlan specific configuration information status VlanStatus Vlan status information"},{"location":"en/reference/kube-ovn-api/#vlanspec","title":"VlanSpec","text":"Property Name Type Description id Int Vlan tag number, in the range of 0~4096 provider String The name of the ProviderNetwork to which the vlan is bound"},{"location":"en/reference/kube-ovn-api/#vlanstatus","title":"VlanStatus","text":"Property Name Type Description subnets []String The list of subnets to which the vlan is bound conditions []VlanCondition Vlan status change information, refer to the beginning of the document for the definition of Condition"},{"location":"en/reference/kube-ovn-api/#providernetwork","title":"ProviderNetwork","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources are kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource will have the value <code>ProviderNetwork</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec ProviderNetworkSpec ProviderNetwork specific configuration information status ProviderNetworkStatus ProviderNetwork status information"},{"location":"en/reference/kube-ovn-api/#providernetworkspec","title":"ProviderNetworkSpec","text":"Property Name Type Description defaultInterface String The name of the NIC interface used by default for this bridge network customInterfaces []CustomInterface The special NIC configuration used by this bridge network excludeNodes []String The names of the nodes that will not be bound to this bridge network exchangeLinkName Bool Whether to exchange the bridge NIC and the corresponding OVS bridge name"},{"location":"en/reference/kube-ovn-api/#custominterface","title":"CustomInterface","text":"Property Name Type Description interface String NIC interface name used for underlay nodes []String List of nodes using the custom NIC interface"},{"location":"en/reference/kube-ovn-api/#providernetworkstatus","title":"ProviderNetworkStatus","text":"Property Name Type Description ready Bool Whether the current bridge network is in the ready state readyNodes []String The name of the node whose bridge network is ready notReadyNodes []String The name of the node whose bridge network is not ready vlans []String The name of the vlan to which the bridge network is bound conditions []ProviderNetworkCondition ProviderNetwork status change information, refer to the beginning of the document for the definition of Condition"},{"location":"en/reference/kube-ovn-api/#vpc-definition","title":"Vpc Definition","text":""},{"location":"en/reference/kube-ovn-api/#vpc","title":"Vpc","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources have this value as kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource will have the value <code>Vpc</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec VpcSpec Vpc specific configuration information status VpcStatus Vpc status information"},{"location":"en/reference/kube-ovn-api/#vpcspec","title":"VpcSpec","text":"Property Name Type Description namespaces []String List of namespaces bound by Vpc staticRoutes []*StaticRoute The static route information configured under Vpc policyRoutes []*PolicyRoute The policy route information configured under Vpc vpcPeerings []*VpcPeering Vpc interconnection information enableExternal Bool Whether vpc is connected to an external switch"},{"location":"en/reference/kube-ovn-api/#staticroute","title":"StaticRoute","text":"Property Name Type Description policy String Routing policy, takes the value of <code>policySrc</code> or <code>policyDst</code> cidr String Routing cidr value nextHopIP String The next hop information of the route"},{"location":"en/reference/kube-ovn-api/#policyroute","title":"PolicyRoute","text":"Property Name Type Description priority Int32 Priority for policy route match String Match expression for policy route action String Action for policy route, the value is in the range of <code>allow</code>, <code>drop</code>, <code>reroute</code> nextHopIP String The next hop of the policy route, separated by commas in the case of ECMP routing"},{"location":"en/reference/kube-ovn-api/#vpcpeering","title":"VpcPeering","text":"Property Name Type Description remoteVpc String Name of the interconnected peering vpc localConnectIP String The local ip for vpc used to connect to peer vpc"},{"location":"en/reference/kube-ovn-api/#vpcstatus","title":"VpcStatus","text":"Property Name Type Description conditions []VpcCondition Vpc status change information, refer to the beginning of the documentation for the definition of Condition standby Bool Whether the vpc creation is complete, the subnet under the vpc needs to wait for the vpc creation to complete other proceeding default Bool Whether it is the default vpc defaultLogicalSwitch String The default subnet under vpc router String The logical-router name for the vpc tcpLoadBalancer String TCP LB information for vpc udpLoadBalancer String UDP LB information for vpc tcpSessionLoadBalancer String TCP Session Hold LB Information for Vpc udpSessionLoadBalancer String UDP session hold LB information for Vpc subnets []String List of subnets for vpc vpcPeerings []String List of peer vpcs for vpc interconnection enableExternal Bool Whether the vpc is connected to an external switch"},{"location":"en/reference/kube-ovn-api/#vpcnatgateway","title":"VpcNatGateway","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources are kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource will have the value <code>VpcNatGateway</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec VpcNatSpec Vpc gateway specific configuration information"},{"location":"en/reference/kube-ovn-api/#vpcnatspec","title":"VpcNatSpec","text":"Property Name Type Description vpc String Vpc name which the vpc gateway belongs to subnet String The name of the subnet to which the gateway pod belongs lanIp String The IP address assigned to the gateway pod selector []String Standard Kubernetes selector match information tolerations []VpcNatToleration Standard Kubernetes tolerance information"},{"location":"en/reference/kube-ovn-api/#vpcnattoleration","title":"VpcNatToleration","text":"Property Name Type Description key String The key information of the taint tolerance operator String Takes the value of <code>Exists</code> or <code>Equal</code> value String The value information of the taint tolerance effect String The effect of the taint tolerance, takes the value of <code>NoExecute</code>, <code>NoSchedule</code>, or <code>PreferNoSchedule</code> tolerationSeconds Int64 The amount of time the pod can continue to run on the node after the taint is added <p>The meaning of the above tolerance fields can be found in the official Kubernetes documentation Taint and Tolerance.</p>"},{"location":"en/reference/kube-ovn-api/#iptableseip","title":"IptablesEIP","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources have this value as kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource have the value <code>IptablesEIP</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec IptablesEipSpec IptablesEIP specific configuration information used by vpc gateway status IptablesEipStatus IptablesEIP status information used by vpc gateway"},{"location":"en/reference/kube-ovn-api/#iptableseipspec","title":"IptablesEipSpec","text":"Property Name Type Description v4ip String IptablesEIP v4 address v6ip String IptablesEIP v6 address macAddress String The assigned mac address, not actually used natGwDp String Vpc gateway name"},{"location":"en/reference/kube-ovn-api/#iptableseipstatus","title":"IptablesEipStatus","text":"Property Name Type Description ready Bool Whether IptablesEIP is configured complete ip String The IP address used by IptablesEIP, currently only IPv4 addresses are supported redo String IptablesEIP crd creation or update time nat String The type of IptablesEIP, either <code>fip</code>, <code>snat</code>, or <code>dnat</code> conditions []IptablesEIPCondition IptablesEIP status change information, refer to the beginning of the documentation for the definition of Condition"},{"location":"en/reference/kube-ovn-api/#iptablesfiprule","title":"IptablesFIPRule","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources have this value as kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource have the value <code>IptablesFIPRule</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec IptablesFIPRuleSpec The IptablesFIPRule specific configuration information used by vpc gateway status IptablesFIPRuleStatus IptablesFIPRule status information used by vpc gateway"},{"location":"en/reference/kube-ovn-api/#iptablesfiprulespec","title":"IptablesFIPRuleSpec","text":"Property Name Type Description eip String Name of the IptablesEIP used for IptablesFIPRule internalIp String The corresponding internal IP address"},{"location":"en/reference/kube-ovn-api/#iptablesfiprulestatus","title":"IptablesFIPRuleStatus","text":"Property Name Type Description ready Bool Whether IptablesFIPRule is configured or not v4ip String The v4 IP address used by IptablesEIP v6ip String The v6 IP address used by IptablesEIP natGwDp String Vpc gateway name redo String IptablesFIPRule crd creation or update time conditions []IptablesFIPRuleCondition IptablesFIPRule status change information, refer to the beginning of the documentation for the definition of Condition"},{"location":"en/reference/kube-ovn-api/#iptablessnatrule","title":"IptablesSnatRule","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources have this value as kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource have the value <code>IptablesSnatRule</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec IptablesSnatRuleSpec The IptablesSnatRule specific configuration information used by the vpc gateway status IptablesSnatRuleStatus IptablesSnatRule status information used by vpc gateway"},{"location":"en/reference/kube-ovn-api/#iptablessnatrulespec","title":"IptablesSnatRuleSpec","text":"Property Name Type Description eip String Name of the IptablesEIP used by IptablesSnatRule internalIp String IptablesSnatRule's corresponding internal IP address"},{"location":"en/reference/kube-ovn-api/#iptablessnatrulestatus","title":"IptablesSnatRuleStatus","text":"Property Name Type Description ready Bool Whether the configuration is complete v4ip String The v4 IP address used by IptablesSnatRule v6ip String The v6 IP address used by IptablesSnatRule natGwDp String Vpc gateway name redo String IptablesSnatRule crd creation or update time conditions []IptablesSnatRuleCondition IptablesSnatRule status change information, refer to the beginning of the documentation for the definition of Condition"},{"location":"en/reference/kube-ovn-api/#iptablesdnatrule","title":"IptablesDnatRule","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources have this value as kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource have the value <code>IptablesDnatRule</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec IptablesDnatRuleSpec The IptablesDnatRule specific configuration information used by vpc gateway status IptablesDnatRuleStatus IptablesDnatRule status information used by vpc gateway"},{"location":"en/reference/kube-ovn-api/#iptablesdnatrulespec","title":"IptablesDnatRuleSpec","text":"Property Name Type Description eip Sting Name of IptablesEIP used by IptablesDnatRule externalPort Sting External port used by IptablesDnatRule protocol Sting Vpc gateway dnat protocol type internalIp Sting Internal IP address used by IptablesDnatRule internalPort Sting Internal port used by IptablesDnatRule"},{"location":"en/reference/kube-ovn-api/#iptablesdnatrulestatus","title":"IptablesDnatRuleStatus","text":"Property Name Type Description ready Bool Whether the configuration is complete v4ip String The v4 IP address used by IptablesDnatRule v6ip String The v6 IP address used by IptablesDnatRule natGwDp String Vpc gateway name redo String IptablesDnatRule crd creation or update time conditions []IptablesDnatRuleCondition IptablesDnatRule Status change information, refer to the beginning of the documentation for the definition of Condition"},{"location":"en/reference/kube-ovn-api/#vpcdns","title":"VpcDns","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources have kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource will have the value <code>VpcDns</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec VpcDnsSpec VpcDns specific configuration information status VpcDnsStatus VpcDns status information"},{"location":"en/reference/kube-ovn-api/#vpcdnsspec","title":"VpcDnsSpec","text":"Property Name Type Description vpc String Name of the vpc where VpcDns is located subnet String The subnet name of the address assigned to the VpcDns pod"},{"location":"en/reference/kube-ovn-api/#vpcdnsstatus","title":"VpcDnsStatus","text":"Property Name Type Description conditions []VpcDnsCondition VpcDns status change information, refer to the beginning of the document for the definition of Condition active Bool Whether VpcDns is in use <p>For detailed documentation on the use of VpcDns, see Customizing VPC DNS.</p>"},{"location":"en/reference/kube-ovn-api/#switchlbrule","title":"SwitchLBRule","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources have this value as kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource will have the value <code>SwitchLBRule</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec SwitchLBRuleSpec SwitchLBRule specific configuration information status SwitchLBRuleStatus SwitchLBRule status information"},{"location":"en/reference/kube-ovn-api/#switchlbrulespec","title":"SwitchLBRuleSpec","text":"Property Name Type Description vip String Vip address of SwitchLBRule namespace String SwitchLBRule's namespace selector []String Standard Kubernetes selector match information sessionAffinity String Standard Kubernetes service sessionAffinity value ports []SlrPort List of SwitchLBRule ports <p>For detailed configuration information of SwitchLBRule, you can refer to Customizing VPC Internal Load Balancing.</p>"},{"location":"en/reference/kube-ovn-api/#slrport","title":"SlrPort","text":"Property Name Type Description name String Port name port Int32 Port number targetPort Int32 Target port of SwitchLBRule protocol String Protocol type"},{"location":"en/reference/kube-ovn-api/#switchlbrulestatus","title":"SwitchLBRuleStatus","text":"Property Name Type Description conditions []SwitchLBRuleCondition SwitchLBRule status change information, refer to the beginning of the document for the definition of Condition ports String Port information service String Name of the service"},{"location":"en/reference/kube-ovn-api/#security-group-and-vip","title":"Security Group and Vip","text":""},{"location":"en/reference/kube-ovn-api/#securitygroup","title":"SecurityGroup","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources are kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource will have a value of <code>SecurityGroup</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec SecurityGroupSpec Security Group specific configuration information status SecurityGroupStatus Security group status information"},{"location":"en/reference/kube-ovn-api/#securitygroupspec","title":"SecurityGroupSpec","text":"Property Name Type Description ingressRules []*SgRule Inbound security group rules egressRules []*SgRule Outbound security group rules allowSameGroupTraffic Bool Whether lsps in the same security group can interoperate and whether traffic rules need to be updated"},{"location":"en/reference/kube-ovn-api/#sgrule","title":"SgRule","text":"Property Name Type Description ipVersion String IP version number, <code>ipv4</code> or <code>ipv6</code> protocol String The value of <code>icmp</code>, <code>tcp</code>, or <code>udp</code> priority Int Acl priority. The value range is 1-200, the smaller the value, the higher the priority. remoteType String The value is either <code>address</code> or <code>securityGroup</code> remoteAddress String The address of the other side remoteSecurityGroup String The name of security group on the other side portRangeMin Int The starting value of the port range, the minimum value is 1. portRangeMax Int The ending value of the port range, the maximum value is 65535. policy String The value is <code>allow</code> or <code>drop</code>"},{"location":"en/reference/kube-ovn-api/#securitygroupstatus","title":"SecurityGroupStatus","text":"Property Name Type Description portGroup String The name of the port-group for the security group allowSameGroupTraffic Bool Whether lsps in the same security group can interoperate, and whether the security group traffic rules need to be updated ingressMd5 String The MD5 value of the inbound security group rule egressMd5 String The MD5 value of the outbound security group rule ingressLastSyncSuccess Bool Whether the last synchronization of the inbound rule was successful egressLastSyncSuccess Bool Whether the last synchronization of the outbound rule was successful"},{"location":"en/reference/kube-ovn-api/#vip","title":"Vip","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources are kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource will have the value <code>Vip</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec VipSpec Vip specific configuration information status VipStatus Vip status information"},{"location":"en/reference/kube-ovn-api/#vipspec","title":"VipSpec","text":"Property Name Type Description namespace String Vip's namespace subnet String Vip's subnet v4ip String Vip IPv4 ip address v6ip String Vip IPv6 ip address macAddress String Vip mac address parentV4ip String Not currently in use parentV6ip String Not currently in use parentMac String Not currently in use attachSubnets []String This field is deprecated and no longer used"},{"location":"en/reference/kube-ovn-api/#vipstatus","title":"VipStatus","text":"Property Name Type Description conditions []VipCondition Vip status change information, refer to the beginning of the documentation for the definition of Condition ready Bool Vip is ready or not v4ip String Vip IPv4 ip address, should be the same as the spec field v6ip String Vip IPv6 ip address, should be the same as the spec field mac String The vip mac address, which should be the same as the spec field pv4ip String Not currently used pv6ip String Not currently used pmac String Not currently used"},{"location":"en/reference/kube-ovn-api/#ovneip","title":"OvnEip","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources are kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource will have the value <code>OvnEip</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec OvnEipSpec OvnEip specific configuration information for default vpc status OvnEipStatus OvnEip status information for default vpc"},{"location":"en/reference/kube-ovn-api/#ovneipspec","title":"OvnEipSpec","text":"Property Name Type Description externalSubnet String OvnEip's subnet name v4ip String OvnEip IP address macAddress String OvnEip Mac address type String OvnEip use type, the value can be <code>fip</code>, <code>snat</code> or <code>lrp</code>"},{"location":"en/reference/kube-ovn-api/#ovneipstatus","title":"OvnEipStatus","text":"Property Name Type Description conditions []OvnEipCondition OvnEip status change information, refer to the beginning of the documentation for the definition of Condition v4ip String The IPv4 ip address used by ovnEip macAddress String Mac address used by ovnEip"},{"location":"en/reference/kube-ovn-api/#ovnfip","title":"OvnFip","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources are kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource will have the value <code>OvnFip</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec OvnFipSpec OvnFip specific configuration information in default vpc status OvnFipStatus OvnFip status information in default vpc"},{"location":"en/reference/kube-ovn-api/#ovnfipspec","title":"OvnFipSpec","text":"Property Name Type Description ovnEip String Name of the bound ovnEip ipName String The IP crd name corresponding to the bound Pod"},{"location":"en/reference/kube-ovn-api/#ovnfipstatus","title":"OvnFipStatus","text":"Property Name Type Description ready Bool OvnFip is ready or not v4Eip String Name of the ovnEip to which ovnFip is bound v4Ip String The ovnEip address currently in use macAddress String OvnFip's configured mac address vpc String The name of the vpc where ovnFip is located conditions []OvnFipCondition OvnFip status change information, refer to the beginning of the document for the definition of Condition"},{"location":"en/reference/kube-ovn-api/#ovnsnatrule","title":"OvnSnatRule","text":"Property Name Type Description apiVersion String Standard Kubernetes version information field, all custom resources have kubeovn.io/v1 kind String Standard Kubernetes resource type field, all instances of this resource will have the value <code>OvnSnatRule</code> metadata ObjectMeta Standard Kubernetes resource metadata information spec OvnSnatRuleSpec OvnSnatRule specific configuration information in default vpc status OvnSnatRuleStatus OvnSnatRule status information in default vpc"},{"location":"en/reference/kube-ovn-api/#ovnsnatrulespec","title":"OvnSnatRuleSpec","text":"Property Name Type Description ovnEip String Name of the ovnEip to which ovnSnatRule is bound vpcSubnet String The name of the subnet configured by ovnSnatRule ipName String The IP crd name corresponding to the ovnSnatRule bound Pod"},{"location":"en/reference/kube-ovn-api/#ovnsnatrulestatus","title":"OvnSnatRuleStatus","text":"Property Name Type Description ready Bool OvnSnatRule is ready or not v4Eip String The ovnEip address to which ovnSnatRule is bound v4IpCidr String The cidr address used to configure snat in the logical-router vpc String The name of the vpc where ovnSnatRule is located conditions []OvnSnatRuleCondition OvnSnatRule status change information, refer to the beginning of the document for the definition of Condition <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/reference/kube-ovn-pinger-args/","title":"Kube-OVN-Pinger args Reference","text":"<p>Based on the Kube-OVN v1.12.0 version, We have compiled the parameters supported by Kube-ovn-pinger, and listed the value types, meanings, and default values of each field defined by the parameters for reference</p>"},{"location":"en/reference/kube-ovn-pinger-args/#args-describeption","title":"Args Describeption","text":"Arg Name Type Description Default Value port Int metrics port 8080 kubeconfig String Path to kubeconfig file with authorization and master location information. If not set use the inCluster token. \"\" ds-namespace String kube-ovn-pinger daemonset namespace \"kube-system\" ds-name String kube-ovn-pinger daemonset name \"kube-ovn-pinger\" interval Int interval seconds between consecutive pings 5 mode String server or job Mode \"server\" exit-code Int exit code when failure happens 0 internal-dns String check dns from pod \"kubernetes.default\" external-dns String check external dns resolve from pod \"\" external-address String check ping connection to an external address \"114.114.114.114\" network-mode String The cni plugin current cluster used \"kube-ovn\" enable-metrics Bool Whether to support metrics query true ovs.timeout Int Timeout on JSON-RPC requests to OVS. 2 system.run.dir String OVS default run directory. \"/var/run/openvswitch\" database.vswitch.name String The name of OVS db. \"Open_vSwitch\" database.vswitch.socket.remote String JSON-RPC unix socket to OVS db. \"unix:/var/run/openvswitch/db.sock\" database.vswitch.file.data.path String OVS db file. \"/etc/openvswitch/conf.db\" database.vswitch.file.log.path String OVS db log file. \"/var/log/openvswitch/ovsdb-server.log\" database.vswitch.file.pid.path String OVS db process id file. \"/var/run/openvswitch/ovsdb-server.pid\" database.vswitch.file.system.id.path String OVS system id file. \"/etc/openvswitch/system-id.conf\" service.vswitchd.file.log.path String OVS vswitchd daemon log file. \"/var/log/openvswitch/ovs-vswitchd.log\" service.vswitchd.file.pid.path String OVS vswitchd daemon process id file. \"/var/run/openvswitch/ovs-vswitchd.pid\" service.ovncontroller.file.log.path String OVN controller daemon log file. \"/var/log/ovn/ovn-controller.log\" service.ovncontroller.file.pid.path String OVN controller daemon process id file. \"/var/run/ovn/ovn-controller.pid\" <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/reference/metrics/","title":"Metrics","text":"<p>This document lists all the monitoring metrics provided by Kube-OVN.</p>"},{"location":"en/reference/metrics/#ovn-monitor","title":"ovn-monitor","text":"<p>OVN status metrics:</p> Type Metric Description Gauge kube_ovn_ovn_status OVN Health Status. The values are: (2) for standby or follower, (1) for active or leader, (0) for unhealthy. Gauge kube_ovn_failed_req_count The number of failed requests to OVN stack. Gauge kube_ovn_log_file_size The size of a log file associated with an OVN component. Gauge kube_ovn_db_file_size The size of a database file associated with an OVN component. Gauge kube_ovn_chassis_info Whether the OVN chassis is up (1) or down (0), together with additional information about the chassis. Gauge kube_ovn_db_status The status of OVN NB/SB DB, (1) for healthy, (0) for unhealthy. Gauge kube_ovn_logical_switch_info The information about OVN logical switch. This metric is always up (1). Gauge kube_ovn_logical_switch_external_id Provides the external IDs and values associated with OVN logical switches. This metric is always up (1). Gauge kube_ovn_logical_switch_port_binding Provides the association between a logical switch and a logical switch port. This metric is always up (1). Gauge kube_ovn_logical_switch_tunnel_key The value of the tunnel key associated with the logical switch. Gauge kube_ovn_logical_switch_ports_num The number of logical switch ports connected to the OVN logical switch. Gauge kube_ovn_logical_switch_port_info The information about OVN logical switch port. This metric is always up (1). Gauge kube_ovn_logical_switch_port_tunnel_key The value of the tunnel key associated with the logical switch port. Gauge kube_ovn_cluster_enabled Is OVN clustering enabled (1) or not (0). Gauge kube_ovn_cluster_role A metric with a constant '1' value labeled by server role. Gauge kube_ovn_cluster_status A metric with a constant '1' value labeled by server status. Gauge kube_ovn_cluster_term The current raft term known by this server. Gauge kube_ovn_cluster_leader_self Is this server consider itself a leader (1) or not (0). Gauge kube_ovn_cluster_vote_self Is this server voted itself as a leader (1) or not (0). Gauge kube_ovn_cluster_election_timer The current election timer value. Gauge kube_ovn_cluster_log_not_committed The number of log entries not yet committed by this server. Gauge kube_ovn_cluster_log_not_applied The number of log entries not yet applied by this server. Gauge kube_ovn_cluster_log_index_start The log entry index start value associated with this server. Gauge kube_ovn_cluster_log_index_next The log entry index next value associated with this server. Gauge kube_ovn_cluster_inbound_connections_total The total number of inbound connections to the server. Gauge kube_ovn_cluster_outbound_connections_total The total number of outbound connections from the server. Gauge kube_ovn_cluster_inbound_connections_error_total The total number of failed inbound connections to the server. Gauge kube_ovn_cluster_outbound_connections_error_total The total number of failed outbound connections from the server."},{"location":"en/reference/metrics/#ovs-monitor","title":"ovs-monitor","text":"<p><code>ovsdb</code> and <code>vswitchd</code> status metrics:</p> Type Metric Description Gauge ovs_status OVS Health Status. The values are: health(1), unhealthy(0). Gauge ovs_info This metric provides basic information about OVS. It is always set to 1. Gauge failed_req_count The number of failed requests to OVS stack. Gauge log_file_size The size of a log file associated with an OVS component. Gauge db_file_size The size of a database file associated with an OVS component. Gauge datapath Represents an existing datapath. This metrics is always 1. Gauge dp_total Represents total number of datapaths on the system. Gauge dp_if Represents an existing datapath interface. This metrics is always 1. Gauge dp_if_total Represents the number of ports connected to the datapath. Gauge dp_flows_total The number of flows in a datapath. Gauge dp_flows_lookup_hit The number of incoming packets in a datapath matching existing flows in the datapath. Gauge dp_flows_lookup_missed The number of incoming packets in a datapath not matching any existing flow in the datapath. Gauge dp_flows_lookup_lost The number of incoming packets in a datapath destined for userspace process but subsequently dropped before reaching userspace. Gauge dp_masks_hit The total number of masks visited for matching incoming packets. Gauge dp_masks_total The number of masks in a datapath. Gauge dp_masks_hit_ratio The average number of masks visited per packet. It is the ration between hit and total number of packets processed by a datapath. Gauge interface Represents OVS interface. This is the primary metric for all other interface metrics. This metrics is always 1. Gauge interface_admin_state The administrative state of the physical network link of OVS interface. The values are: down(0), up(1), other(2). Gauge interface_link_state The state of the physical network link of OVS interface. The values are: down(0), up(1), other(2). Gauge interface_mac_in_use The MAC address in use by OVS interface. Gauge interface_mtu The currently configured MTU for OVS interface. Gauge interface_of_port Represents the OpenFlow port ID associated with OVS interface. Gauge interface_if_index Represents the interface index associated with OVS interface. Gauge interface_tx_packets Represents the number of transmitted packets by OVS interface. Gauge interface_tx_bytes Represents the number of transmitted bytes by OVS interface. Gauge interface_rx_packets Represents the number of received packets by OVS interface. Gauge interface_rx_bytes Represents the number of received bytes by OVS interface. Gauge interface_rx_crc_err Represents the number of CRC errors for the packets received by OVS interface. Gauge interface_rx_dropped Represents the number of input packets dropped by OVS interface. Gauge interface_rx_errors Represents the total number of packets with errors received by OVS interface. Gauge interface_rx_frame_err Represents the number of frame alignment errors on the packets received by OVS interface. Gauge interface_rx_missed_err Represents the number of packets with RX missed received by OVS interface. Gauge interface_rx_over_err Represents the number of packets with RX overrun received by OVS interface. Gauge interface_tx_dropped Represents the number of output packets dropped by OVS interface. Gauge interface_tx_errors Represents the total number of transmit errors by OVS interface. Gauge interface_collisions Represents the number of collisions on OVS interface."},{"location":"en/reference/metrics/#kube-ovn-pinger","title":"kube-ovn-pinger","text":"<p>Network quality related metrics:</p> Type Metric Description Gauge pinger_ovs_up If the ovs on the node is up Gauge pinger_ovs_down If the ovs on the node is down Gauge pinger_ovn_controller_up If the ovn_controller on the node is up Gauge pinger_ovn_controller_down If the ovn_controller on the node is down Gauge pinger_inconsistent_port_binding The number of mismatch port bindings between ovs and ovn-sb Gauge pinger_apiserver_healthy If the apiserver request is healthy on this node Gauge pinger_apiserver_unhealthy If the apiserver request is unhealthy on this node Histogram pinger_apiserver_latency_ms The latency ms histogram the node request apiserver Gauge pinger_internal_dns_healthy If the internal dns request is unhealthy on this node Gauge pinger_internal_dns_unhealthy If the internal dns request is unhealthy on this node Histogram pinger_internal_dns_latency_ms The latency ms histogram the node request internal dns Gauge pinger_external_dns_health If the external dns request is healthy on this node Gauge pinger_external_dns_unhealthy If the external dns request is unhealthy on this node Histogram pinger_external_dns_latency_ms The latency ms histogram the node request external dns Histogram pinger_pod_ping_latency_ms The latency ms histogram for pod peer ping Gauge pinger_pod_ping_lost_total The lost count for pod peer ping Gauge pinger_pod_ping_count_total The total count for pod peer ping Histogram pinger_node_ping_latency_ms The latency ms histogram for pod ping node Gauge pinger_node_ping_lost_total The lost count for pod ping node Gauge pinger_node_ping_count_total The total count for pod ping node Histogram pinger_external_ping_latency_ms The latency ms histogram for pod ping external address Gauge pinger_external_lost_total The lost count for pod ping external address"},{"location":"en/reference/metrics/#kube-ovn-controller","title":"kube-ovn-controller","text":"<p><code>kube-ovn-controller</code> status metrics\uff1a</p> Type Metric Description Histogram rest_client_request_latency_seconds Request latency in seconds. Broken down by verb and URL Counter rest_client_requests_total Number of HTTP requests, partitioned by status code, method, and host Counter lists_total Total number of API lists done by the reflectors Summary list_duration_seconds How long an API list takes to return and decode for the reflectors Summary items_per_list How many items an API list returns to the reflectors Counter watches_total Total number of API watches done by the reflectors Counter short_watches_total Total number of short API watches done by the reflectors Summary watch_duration_seconds How long an API watch takes to return and decode for the reflectors Summary items_per_watch How many items an API watch returns to the reflectors Gauge last_resource_version Last resource version seen for the reflectors Histogram ovs_client_request_latency_milliseconds The latency histogram for ovs request Gauge subnet_available_ip_count The available num of ip address in subnet Gauge subnet_used_ip_count The used num of ip address in subnet"},{"location":"en/reference/metrics/#kube-ovn-cni","title":"kube-ovn-cni","text":"<p><code>kube-ovn-cni</code> status metrics:</p> Type Metric Description Histogram cni_op_latency_seconds The latency seconds for cni operations Counter cni_wait_address_seconds_total Latency that cni wait controller to assign an address Counter cni_wait_connectivity_seconds_total Latency that cni wait address ready in overlay network Counter cni_wait_route_seconds_total Latency that cni wait controller to add routed annotation to pod Histogram rest_client_request_latency_seconds Request latency in seconds. Broken down by verb and URL Counter rest_client_requests_total Number of HTTP requests, partitioned by status code, method, and host Counter lists_total Total number of API lists done by the reflectors Summary list_duration_seconds How long an API list takes to return and decode for the reflectors Summary items_per_list How many items an API list returns to the reflectors Counter watches_total Total number of API watches done by the reflectors Counter short_watches_total Total number of short API watches done by the reflectors Summary watch_duration_seconds How long an API watch takes to return and decode for the reflectors Summary items_per_watch How many items an API watch returns to the reflectors Gauge last_resource_version Last resource version seen for the reflectors Histogram ovs_client_request_latency_milliseconds The latency histogram for ovs request <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/reference/ovs-ovn-customized/","title":"OVS/OVN Customization","text":"<p>Upstream OVN/OVS was originally designed with the goal of a general purpose SDN controller and data plane. Due to some specific usage of the Kubernetes network,Kube-OVN only focused on part of the features. In order to achieve better performance, stability and specific features, Kube-OVN has made some modifications to the upstream OVN/OVS. Users using their own OVN/OVS with Kube-OVN controllers need to be aware of the possible impact of the following changes:</p> <p>Did not merge into the upstream modification.</p> <ul> <li>38df6fa3f7 Adjust the election timer to avoid large-scale cluster election jitter.</li> <li>d4888c4e75 add fdb update logging.</li> <li>d4888c4e75 fdb: fix mac learning in environments with hairpin enabled.</li> <li>9a81b91368 ovsdb-tool: add optional server id parameter for \"join-cluster\" command.</li> <li>0700cb90f9 Destination non-service traffic bypasses conntrack to improve performance on a particular data path.</li> <li>c48049a64f ECMP algorithm is adjusted from <code>dp_hash</code> to <code>hash</code> to avoid the hash error problem in some kernels.</li> <li>64383c14a9 Fix kernel Crash issue under Windows.</li> <li>08a95db2ca Support for github action builds on Windows.</li> <li>680e77a190 Windows uses tcp listening by default.</li> <li>05e57b3227 add support for windows.</li> <li>0181b68be1 br-int controller: listen on 127.0.0.1:6653 by default.</li> <li>b3801ecb73 modify src route priority.</li> <li>977e569539 fix reaching resubmit limit in underlay.</li> <li>45a4a22161 ovn-nbctl: do not remove LB if vips is empty.</li> <li>540592b9ff Replaces the Mac address as the destination address after DNAT to reduce additional performance overhead.</li> <li>10972d9632 Fix vswitchd ofport_usage memory leak.</li> </ul> <p>Merged into upstream modification:</p> <ul> <li>20626ea909 Multicast traffic bypasses LB and ACL processing stages to improve specific data path performance.</li> <li>a2d9ff3ccd Deb build adds compile optimization options.</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/reference/tunnel-protocol/","title":"Tunnel Protocol Selection","text":"<p>Kube-OVN uses OVN/OVS as the data plane implementation and currently supports <code>Geneve</code>, <code>Vxlan</code> and <code>STT</code> tunnel encapsulation protocols. These three protocols differ in terms of functionality, performance and ease of use. This document will describe the differences in the use of the three protocols so that users can choose according to their situation.</p>"},{"location":"en/reference/tunnel-protocol/#geneve","title":"Geneve","text":"<p>The <code>Geneve</code> protocol is the default tunneling protocol selected during Kube-OVN deployment and is also the default recommended tunneling protocol for OVN. This protocol is widely supported in the kernel and can be accelerated using the generic offload capability of modern NICs. Since <code>Geneve</code> has a variable header, it is possible to use 24bit space to mark different datapaths users can create a larger number of virtual networks.</p> <p>If you are using Mellanox or Corigine SmartNIC OVS offload, <code>Geneve</code> requires a higher kernel version. Upstream kernel of 5.4 or higher, or other compatible kernels that backports this feature.</p> <p>Due to the use of UDP encapsulation, this protocol does not make good use of the TCP-related offloads of modern NICs when handling TCP over UDP, and consumes more CPU resources when handling large packets.</p>"},{"location":"en/reference/tunnel-protocol/#vxlan","title":"Vxlan","text":"<p><code>Vxlan</code> is a recently supported protocol in the upstream OVN, which is widely supported in the kernel and can be accelerated using the common offload capabilities of modern NICs. Due to the limited length of the protocol header and the additional space required for OVN orchestration, there is a limit to the number of datapaths that can be created, with a maximum of 4096 datapaths and a maximum of 4096 ports under each datapath. Also, <code>inport</code>-based ACLs are not supported due to header length limitations.</p> <p><code>Vxlan</code> offloading is supported in common kernels if using Mellanox or Corigine SmartNIC.</p> <p>Due to the use of UDP encapsulation, this protocol does not make good use of the TCP-related offloads of modern NICs when handling TCP over UDP, and consumes more CPU resources when handling large packets.</p>"},{"location":"en/reference/tunnel-protocol/#stt","title":"STT","text":"<p>The <code>STT</code> protocol is an early tunneling protocol supported by the OVN that uses TCP-like headers to take advantage of the TCP offload capabilities common to modern NICs and significantly increase TCP throughput. The protocol also has a long header to support full OVN capabilities and large-scale datapaths.</p> <p>This protocol is not supported in the kernel. To use it, you need to compile an additional OVS kernel module and recompile the new version of the kernel module when upgrading the kernel.</p> <p>This protocol is not currently supported by the SmartNic and cannot use the offloading capability of OVS offloading.</p>"},{"location":"en/reference/tunnel-protocol/#references","title":"References","text":"<ul> <li>https://ipwithease.com/vxlan-vs-geneve-understand-the-difference/</li> <li>OVN FAQ</li> <li>What is Geneve</li> </ul> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/reference/underlay-topology/","title":"Underlay Traffic Topology","text":"<p>This document describes the forwarding path of traffic in Underlay mode under different scenarios.</p>"},{"location":"en/reference/underlay-topology/#pods-in-same-node-and-same-subnet","title":"Pods in Same Node and Same Subnet","text":"<p>Internal logical switches exchange packets directly, without access to the external network.</p> <p></p>"},{"location":"en/reference/underlay-topology/#pods-in-different-nodes-and-same-subnet","title":"Pods in Different Nodes and Same Subnet","text":"<p>Packets enter the physic switch via the node NIC and are exchanged by the physic switch.</p> <p></p>"},{"location":"en/reference/underlay-topology/#pods-in-same-node-and-different-subnets","title":"Pods in Same Node and Different Subnets","text":"<p>Packets enter the physic network via the node NIC and are exchanged and routed and forwarded by physic switches and routers.</p> <p></p> <p>Here br-provider-1 and br-provider-2 can be the same OVS bridge\uff0cmultiple subnet can share a Provider Network\u3002</p>"},{"location":"en/reference/underlay-topology/#pods-in-different-nodes-and-different-subnets","title":"Pods in Different Nodes and Different Subnets","text":"<p>Packets enter the physic network via the node NIC and are exchanged and routed and forwarded by physic switches and routers.</p> <p></p>"},{"location":"en/reference/underlay-topology/#access-to-external","title":"Access to External","text":"<p>Packets enter the physic network via the node NIC and are exchanged and routed and forwarded by physic switches and routers.</p> <p></p> <p>The communication between nodes and Pods follows the same logic.</p>"},{"location":"en/reference/underlay-topology/#overview-without-vlan-tag","title":"Overview without Vlan Tag","text":""},{"location":"en/reference/underlay-topology/#overview-with-vlan-tag","title":"Overview with Vlan Tag","text":""},{"location":"en/reference/underlay-topology/#pod-visit-service-ip","title":"Pod visit Service IP","text":"<p>Kube-OVN configures load balancing for each Kubernetes Service on a logical switch on each subnet. When a Pod accesses other Pods by accessing the Service IP, a network packet is constructed with the Service IP as the destination address and the MAC address of the gateway as the destination MAC address. After the network packet enters the logical switch, load balancing will intercept and DNAT the network packet to modify the destination IP and port to the IP and port of one of the Endpoint corresponding to the Service. Since the logical switch does not modify the Layer 2 destination MAC address of the network packet, the network packet will still be delivered to the physic gateway after entering the physic switch, and the physic gateway will be required to forward the network packet.</p>"},{"location":"en/reference/underlay-topology/#service-backend-is-the-same-node-and-same-subnet-pod","title":"Service Backend is the Same Node and Same Subnet Pod","text":""},{"location":"en/reference/underlay-topology/#service-backend-is-the-same-node-and-different-subnets-pod","title":"Service Backend is the Same Node and Different Subnets Pod","text":"<p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/start/one-step-install/","title":"One-Click Installation","text":"<p>Kube-OVN provides a one-click installation script to help you quickly install a highly available, production-ready Kube-OVN container network with Overlay networking by default.</p> <p>Helm Chart installation is supported since Kube-OVN v1.12.0, and the default deployment is Overlay networking.</p> <p>If you need Underlay/Vlan networking as the default container network\uff0cplease read Underlay Installation</p> <p>Before installation please read Prerequisites first to make sure the environment is ready.</p>"},{"location":"en/start/one-step-install/#script-installation","title":"Script Installation","text":""},{"location":"en/start/one-step-install/#download-the-installation-script","title":"Download the installation script","text":"<p>We recommend using the stable release version for production environments, please use the following command to download:</p> <pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/images/install.sh\n</code></pre> <p>If you are interested in the latest features of the master branch, please use the following command to download:</p> <pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/images/install.sh\n</code></pre>"},{"location":"en/start/one-step-install/#modify-configuration-options","title":"Modify Configuration Options","text":"<p>Open the script using the editor and change the following variables to the expected:</p> <pre><code>REGISTRY=\"kubeovn\"                     # Image Repo \nVERSION=\"v1.13.0\"                      # Image Tag\nPOD_CIDR=\"10.16.0.0/16\"                # Default subnet CIDR don't overlay with SVC/NODE/JOIN CIDR\nSVC_CIDR=\"10.96.0.0/12\"                # Be consistent with apiserver's service-cluster-ip-range\nJOIN_CIDR=\"100.64.0.0/16\"              # Pod/Host communication Subnet CIDR, don't overlay with SVC/NODE/POD CIDR\nLABEL=\"node-role.kubernetes.io/master\" # The node label to deploy OVN DB\nIFACE=\"\"                               # The name of the host NIC used by the container network, or if empty use the NIC that host Node IP in Kubernetes\nTUNNEL_TYPE=\"geneve\"                   # Tunnel protocol\uff0cavailable options: geneve, vxlan or stt. stt requires compilation of ovs kernel module\n</code></pre> <p>You can also use regular expression to math NIC names\uff0csuch as <code>IFACE=enp6s0f0,eth.*</code>.</p>"},{"location":"en/start/one-step-install/#run-the-script","title":"Run the Script","text":"<p><code>bash install.sh</code></p> <p>Wait Kube-OVN ready.</p>"},{"location":"en/start/one-step-install/#helm-chart-installation","title":"Helm Chart Installation","text":"<p>Since the installation of Kube-OVN requires setting some parameters, to install Kube-OVN using Helm, you need to follow the steps below.</p>"},{"location":"en/start/one-step-install/#view-the-node-ip-address","title":"View the node IP address","text":"<pre><code>$ kubectl get node -o wide\nNAME                     STATUS     ROLES           AGE   VERSION   INTERNAL-IP   EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION      CONTAINER-RUNTIME\nkube-ovn-control-plane   NotReady   control-plane   20h   v1.26.0   172.18.0.3    &lt;none&gt;        Ubuntu 22.04.1 LTS   5.10.104-linuxkit   containerd://1.6.9\nkube-ovn-worker          NotReady   &lt;none&gt;          20h   v1.26.0   172.18.0.2    &lt;none&gt;        Ubuntu 22.04.1 LTS   5.10.104-linuxkit   containerd://1.6.9\n</code></pre>"},{"location":"en/start/one-step-install/#remove-cluster-master-node-taint","title":"Remove cluster master node taint","text":"<pre><code>$ kubectl taint node kube-ovn-control-plane node-role.kubernetes.io/control-plane:NoSchedule-\nnode/kube-ovn-control-plane untainted\n</code></pre> <p>This step can be skipped if you are sure that you do not need to schedule the pod at the master node.</p>"},{"location":"en/start/one-step-install/#add-label-to-node","title":"Add label to node","text":"<pre><code>$ kubectl label node -lbeta.kubernetes.io/os=linux kubernetes.io/os=linux --overwrite\nnode/kube-ovn-control-plane not labeled\nnode/kube-ovn-worker not labeled\n\n$ kubectl label node -lnode-role.kubernetes.io/control-plane kube-ovn/role=master --overwrite\nnode/kube-ovn-control-plane labeled\n\n# The following labels are used for the installation of dpdk images and can be ignored in non-dpdk cases\n$ kubectl label node -lovn.kubernetes.io/ovs_dp_type!=userspace ovn.kubernetes.io/ovs_dp_type=kernel --overwrite\nnode/kube-ovn-control-plane labeled\nnode/kube-ovn-worker labeled\n</code></pre>"},{"location":"en/start/one-step-install/#add-helm-repo-information","title":"Add Helm Repo information","text":"<pre><code>$ helm repo add kubeovn https://kubeovn.github.io/kube-ovn/\n\"kubeovn\" has been added to your repositories\n\n$ helm repo list\nNAME            URL\nkubeovn         https://kubeovn.github.io/kube-ovn/\n\n$ helm search repo kubeovn\nNAME                CHART VERSION   APP VERSION DESCRIPTION\nkubeovn/kube-ovn    0.1.0           1.12.0      Helm chart for Kube-OVN\n</code></pre>"},{"location":"en/start/one-step-install/#run-helm-install-to-install-kube-ovn","title":"Run helm install to install Kube-OVN","text":"<p>The Node0IP, Node1IP, and Node2IP parameters are the IP addresses of the cluster master nodes, respectively. For other parameters, you can refer to the variable definitions in the values.yaml file.</p> <pre><code># Single master node environment install\n$ helm install kube-ovn kubeovn/kube-ovn --set MASTER_NODES=${Node0IP}\n\n# Using the node information above as an example, execute the install command\n$ helm install kube-ovn kubeovn/kube-ovn --set MASTER_NODES=172.18.0.3\nNAME: kube-ovn\nLAST DEPLOYED: Fri Mar 31 12:43:43 2023\nNAMESPACE: default\nSTATUS: deployed\nREVISION: 1\nTEST SUITE: None\n\n# Highly Available Cluster Installation\n$ helm install kube-ovn kubeovn/kube-ovn --set MASTER_NODES=${Node0IP},${Node1IP},${Node2IP}, --set replicaCount=3\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/start/prepare/","title":"Prerequisites","text":"<p>Kube-OVN is a CNI-compliant network system that depends on the Kubernetes environment and the corresponding kernel network module for its operation. Below are the operating system and software versions tested, the environment configuration and the ports that need to be opened.</p>"},{"location":"en/start/prepare/#software-version","title":"Software Version","text":"<ul> <li>Kubernetes &gt;= 1.23.</li> <li>Docker &gt;= 1.12.6, Containerd &gt;= 1.3.4.</li> <li>OS: CentOS 7/8, Ubuntu 16.04/18.04/20.04.</li> <li>For other Linux distributions, please make sure <code>geneve</code>, <code>openvswitch</code>, <code>ip_tables</code> and <code>iptable_nat</code> kernel modules exist.</li> </ul> <p>Attention\uff1a</p> <ol> <li>For CentOS kernel version 3.10.0-862 bug exists in <code>netfilter</code> modules that lead Kube-OVN embed nat and lb failure.Please update kernel and check Floating IPs broken after kernel upgrade to Centos/RHEL 7.5 - DNAT not working.</li> <li>Kernel version 4.18.0-372.9.1.el8.x86_64 in Rocky Linux 8.6 has a TCP connection problem TCP connection failed in Rocky Linux 8.6\uff0cplease update kernel to 4.18.0-372.13.1.el8_6.x86_64 or later\u3002</li> <li>For kernel version 4.4, the related <code>openvswitch</code> module has some issues for ct\uff0cplease update kernel version or manually compile <code>openvswitch</code> kernel module.</li> <li>When building Geneve tunnel IPv6 in kernel should be enabled\uff0ccheck the kernel bootstrap options with <code>cat /proc/cmdline</code>.Check Geneve tunnels don't work when ipv6 is disabled for the detail bug info.</li> </ol>"},{"location":"en/start/prepare/#environment-setup","title":"Environment Setup","text":"<ul> <li>Kernel should enable IPv6, if kernel bootstrap options contain <code>ipv6.disable=1</code>, it should be set to <code>0</code>.</li> <li><code>kube-proxy</code> works, Kube-OVN can visit <code>kube-apiserver</code> from Service ClusterIP.</li> <li>Make sure kubelet enabled <code>CNI</code> and find cni-bin and cni-conf in default directories, kubelet bootstrap options should contain <code>--network-plugin=cni --cni-bin-dir=/opt/cni/bin --cni-conf-dir=/etc/cni/net.d</code>.</li> <li>Make sure no other CNI installed or has been removed\uff0ccheck if any config files still exist in<code>/etc/cni/net.d/</code>.</li> </ul>"},{"location":"en/start/prepare/#ports-need-open","title":"Ports Need Open","text":"Component Port Usage ovn-central 6641/tcp, 6642/tcp, 6643/tcp, 6644/tcp ovn-db and raft server listen ports ovs-ovn Geneve 6081/udp, STT 7471/tcp, Vxlan 4789/udp tunnel ports kube-ovn-controller 10660/tcp metrics port kube-ovn-daemon 10665/tcp metrics port kube-ovn-monitor 10661/tcp metrics port <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/start/sealos-install/","title":"One-Click Deployment of Kubernetes and Kube-OVN with sealos","text":"<p>sealos, a distribution of Kubernetes, helps users quickly initialize a container cluster from scratch. By using sealos, users can deploy a Kubernetes cluster with Kube-OVN installed in minutes with a single command.</p>"},{"location":"en/start/sealos-install/#download-sealos","title":"Download sealos","text":"AMD64 ARM64 <pre><code>wget https://github.com/labring/sealos/releases/download/v4.0.0/sealos_4.0.0_linux_amd64.tar.gz \\\n&amp;&amp; tar zxvf sealos_4.0.0_linux_amd64.tar.gz sealos &amp;&amp; chmod +x sealos &amp;&amp; mv sealos /usr/bin\n</code></pre> <pre><code>wget https://github.com/labring/sealos/releases/download/v4.0.0/sealos_4.0.0_linux_arm64.tar.gz \\\n&amp;&amp; tar zxvf sealos_4.0.0_linux_arm64.tar.gz sealos &amp;&amp; chmod +x sealos &amp;&amp; mv sealos /usr/bin\n</code></pre>"},{"location":"en/start/sealos-install/#deploy-kubernetes-and-kube-ovn","title":"Deploy Kubernetes and Kube-OVN","text":"<pre><code>```bash\nsealos run labring/kubernetes:v1.24.3 labring/kube-ovn:v1.10.5 \\\n  --masters [masters ips seperated by comma] \\\n  --nodes [nodes ips seperated by comma] -p [your-ssh-passwd]\n```\n</code></pre>"},{"location":"en/start/sealos-install/#wait-to-finish","title":"Wait to finish","text":"<pre><code>```bash\n[Step 6/6] Finish\n\n                    ,,,,\n                    ,::,\n                   ,,::,,,,\n            ,,,,,::::::::::::,,,,,\n         ,,,::::::::::::::::::::::,,,\n       ,,::::::::::::::::::::::::::::,,\n     ,,::::::::::::::::::::::::::::::::,,\n    ,::::::::::::::::::::::::::::::::::::,\n   ,:::::::::::::,,   ,,:::::,,,::::::::::,\n ,,:::::::::::::,       ,::,     ,:::::::::,\n ,:::::::::::::,   :x,  ,::  :,   ,:::::::::,\n,:::::::::::::::,  ,,,  ,::, ,,  ,::::::::::,\n,:::::::::::::::::,,,,,,:::::,,,,::::::::::::,    ,:,   ,:,            ,xx,                            ,:::::,   ,:,     ,:: :::,    ,x\n,::::::::::::::::::::::::::::::::::::::::::::,    :x: ,:xx:        ,   :xx,                          :xxxxxxxxx, :xx,   ,xx:,xxxx,   :x\n,::::::::::::::::::::::::::::::::::::::::::::,    :xxxxx:,  ,xx,  :x:  :xxx:x::,  ::xxxx:           :xx:,  ,:xxx  :xx, ,xx: ,xxxxx:, :x\n,::::::::::::::::::::::::::::::::::::::::::::,    :xxxxx,   :xx,  :x:  :xxx,,:xx,:xx:,:xx, ,,,,,,,,,xxx,    ,xx:   :xx:xx:  ,xxx,:xx::x\n,::::::,,::::::::,,::::::::,,:::::::,,,::::::,    :x:,xxx:  ,xx,  :xx  :xx:  ,xx,xxxxxx:, ,xxxxxxx:,xxx:,  ,xxx,    :xxx:   ,xxx, :xxxx\n,::::,    ,::::,   ,:::::,   ,,::::,    ,::::,    :x:  ,:xx,,:xx::xxxx,,xxx::xx: :xx::::x: ,,,,,,   ,xxxxxxxxx,     ,xx:    ,xxx,  :xxx\n,::::,    ,::::,    ,::::,    ,::::,    ,::::,    ,:,    ,:,  ,,::,,:,  ,::::,,   ,:::::,            ,,:::::,        ,,      :x:    ,::\n,::::,    ,::::,    ,::::,    ,::::,    ,::::,\n ,,,,,    ,::::,    ,::::,    ,::::,    ,:::,             ,,,,,,,,,,,,,\n          ,::::,    ,::::,    ,::::,    ,:::,        ,,,:::::::::::::::,\n          ,::::,    ,::::,    ,::::,    ,::::,  ,,,,:::::::::,,,,,,,:::,\n          ,::::,    ,::::,    ,::::,     ,::::::::::::,,,,,\n           ,,,,     ,::::,     ,,,,       ,,,::::,,,,\n                    ,::::,\n                    ,,::,\n\nThanks for choosing Kube-OVN!\nFor more advanced features, please read https://github.com/kubeovn/kube-ovn#documents\nIf you have any question, please file an issue https://github.com/kubeovn/kube-ovn/issues/new/choose\n2022-08-10T16:31:34 info succeeded in creating a new cluster, enjoy it!\n2022-08-10T16:31:34 info\n      ___           ___           ___           ___       ___           ___\n     /\\  \\         /\\  \\         /\\  \\         /\\__\\     /\\  \\         /\\  \\\n    /::\\  \\       /::\\  \\       /::\\  \\       /:/  /    /::\\  \\       /::\\  \\\n   /:/\\ \\  \\     /:/\\:\\  \\     /:/\\:\\  \\     /:/  /    /:/\\:\\  \\     /:/\\ \\  \\\n  _\\:\\~\\ \\  \\   /::\\~\\:\\  \\   /::\\~\\:\\  \\   /:/  /    /:/  \\:\\  \\   _\\:\\~\\ \\  \\\n /\\ \\:\\ \\ \\__\\ /:/\\:\\ \\:\\__\\ /:/\\:\\ \\:\\__\\ /:/__/    /:/__/ \\:\\__\\ /\\ \\:\\ \\ \\__\\\n \\:\\ \\:\\ \\/__/ \\:\\~\\:\\ \\/__/ \\/__\\:\\/:/  / \\:\\  \\    \\:\\  \\ /:/  / \\:\\ \\:\\ \\/__/\n  \\:\\ \\:\\__\\    \\:\\ \\:\\__\\        \\::/  /   \\:\\  \\    \\:\\  /:/  /   \\:\\ \\:\\__\\\n   \\:\\/:/  /     \\:\\ \\/__/        /:/  /     \\:\\  \\    \\:\\/:/  /     \\:\\/:/  /\n    \\::/  /       \\:\\__\\         /:/  /       \\:\\__\\    \\::/  /       \\::/  /\n     \\/__/         \\/__/         \\/__/         \\/__/     \\/__/         \\/__/\n\n                  Website :https://www.sealos.io/\n                  Address :github.com/labring/sealos\n```\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/start/underlay/","title":"Underlay Installation","text":"<p>By default, the default subnet uses Geneve to encapsulate cross-host traffic, and build an overlay network on top of the infrastructure.</p> <p>For the case that you want the container network to use the physical network address directly, you can set the default subnet of Kube-OVN to work in Underlay mode, which can directly assign the address resources in the physical network to the containers, achieving better performance and connectivity with the physical network.</p> <p></p>"},{"location":"en/start/underlay/#limitation","title":"Limitation","text":"<p>Since the container network in this mode uses physical network directly for L2 packet forwarding, L3 functions such as SNAT/EIP, distributed gateway/centralized gateway in Overlay mode cannot be used. VPC level isolation is also not available for underlay subnet.</p>"},{"location":"en/start/underlay/#comparison-with-macvlan","title":"Comparison with Macvlan","text":"<p>The Underlay mode of Kube-OVN is very similar to the Macvlan, with the following major differences in functionality and performance:</p> <ol> <li>Macvlan performs better in terms of throughput and latency performance metrics due to its shorter kernel path and the fact that it does not require OVS for packet processing.</li> <li>Kube-OVN provides arp-proxy functionality through flow tables to mitigate the risk of arp broadcast storms on large-scale networks.</li> <li>Since Macvlan works at the bottom of the kernel and bypasses the host netfilter, Service and NetworkPolicy functionality requires additional development.    Kube-OVN provides Service and NetworkPolicy capabilities through the OVS flow table.</li> <li>Kube-OVN Underlay mode provides additional features such as address management, fixed IP and QoS compared to Macvlan.</li> </ol>"},{"location":"en/start/underlay/#environment-requirements","title":"Environment Requirements","text":"<p>In Underlay mode, the OVS will bridge a node NIC to the OVS bridge and send packets directly through that node NIC, relying on the underlying network devices for L2/L3 level forwarding capabilities. You need to configure the corresponding gateway, Vlan and security policy in the underlying network device in advance.</p> <ol> <li>For OpenStack VM environments, you need to turn off <code>PortSecurity</code> on the corresponding network port.</li> <li>For VMware vSwitch networks, <code>MAC Address Changes</code>, <code>Forged Transmits</code> and <code>Promiscuous Mode Operation</code> should be set to <code>allow</code>.</li> <li>For Hyper-V virtualization,  <code>MAC Address Spoofing</code> should be enabled in VM nic advanced features.</li> <li>Public clouds, such as AWS, GCE, AliCloud, etc., do not support user-defined Mac, so they cannot support Underlay mode network. In this scenario, if you want to use Underlay, it is recommended to use the VPC-CNI provided by the corresponding public cloud vendor..</li> <li>The network interface that is bridged into ovs can not be type of Linux Bridge.</li> </ol> <p>For management and container networks using the same NIC, Kube-OVN will transfer the NIC's Mac address, IP address, route, and MTU to the corresponding OVS Bridge to support single NIC deployment of Underlay networks. OVS Bridge name format is <code>br-PROVIDER_NAME</code>\uff0c<code>PROVIDER_NAME</code> is the name of <code>ProviderNetwork</code> (Default: provider).</p>"},{"location":"en/start/underlay/#specify-network-mode-when-deploying","title":"Specify Network Mode When Deploying","text":"<p>This deployment mode sets the default subnet to Underlay mode, and all Pods with no subnet specified will run in the Underlay network by default.</p>"},{"location":"en/start/underlay/#download-script","title":"Download Script","text":"<pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/images/install.sh\n</code></pre>"},{"location":"en/start/underlay/#modify-configuration-options","title":"Modify Configuration Options","text":"<pre><code>ENABLE_ARP_DETECT_IP_CONFLICT # disable vlan arp conflict detection if necessary\nNETWORK_TYPE                  # set to vlan\nVLAN_INTERFACE_NAME           # set to the NIC that carries the Underlay traffic, e.g. eth1\nVLAN_ID                       # The VLAN Tag need to be added\uff0cif set 0 no vlan tag will be added\nPOD_CIDR                      # The Underlay network CIDR\uff0c e.g. 192.168.1.0/24\nPOD_GATEWAY                   # Underlay physic gateway address, e.g. 192.168.1.1\nEXCLUDE_IPS                   # Exclude ranges to avoid conflicts between container network and IPs already in use on the physical network, e.g. 192.168.1.1..192.168.1.100\nENABLE_LB                     # If Underlay Subnet needs to visit Service set it to true\nEXCHANGE_LINK_NAME            # If swap the names of the OVS bridge and the bridge interface under the default provider-network. Default to false.\nLS_DNAT_MOD_DL_DST            # If DNAT translate MAC addresses to accelerate service access. Default to true.\n</code></pre>"},{"location":"en/start/underlay/#run-the-script","title":"Run the Script","text":"<pre><code>bash install.sh\n</code></pre>"},{"location":"en/start/underlay/#dynamically-create-underlay-networks-via-crd","title":"Dynamically Create Underlay Networks via CRD","text":"<p>This approach dynamically creates an Underlay subnet that Pod can use after installation.</p>"},{"location":"en/start/underlay/#create-providernetwork","title":"Create ProviderNetwork","text":"<p>ProviderNetwork provides the abstraction of host NIC to physical network mapping, unifies the management of NICs belonging to the same network, and solves the configuration problems in complex environments with multiple NICs on the same machine, inconsistent NIC names and inconsistent corresponding Underlay networks.</p> <p>Create ProviderNetwork as below:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: ProviderNetwork\nmetadata:\n  name: net1\nspec:\n  defaultInterface: eth1\n  customInterfaces:\n    - interface: eth2\n      nodes:\n        - node1\n  excludeNodes:\n    - node2\n</code></pre> <p>Note: The length of the ProviderNetwork resource name must not exceed 12.</p> <ul> <li><code>defaultInterface</code>: The default node NIC name. When the ProviderNetwork is successfully created, an OVS bridge named br-net1 (in the format <code>br-NAME</code>) is created in each node (except excludeNodes) and the specified node NIC is bridged to this bridge.</li> <li><code>customInterfaces</code>: Optionally, you can specify the NIC to be used for a specific node.</li> <li><code>excludeNodes</code>: Optional, to specify nodes that do not bridge the NIC. Nodes in this list will be added with the <code>net1.provider-network.ovn.kubernetes.io/exclude=true</code> tag.</li> </ul> <p>Other nodes will be added with the following tags:</p> Key Value Description net1.provider-network.ovn.kubernetes.io/ready true bridge work finished, ProviderNetwork is ready on this node net1.provider-network.ovn.kubernetes.io/interface eth1 The name of the bridged NIC in the node. net1.provider-network.ovn.kubernetes.io/mtu 1500 MTU of bridged NIC in node <p>If an IP has been configured on the node NIC, the IP address and the route on the NIC are transferred to the corresponding OVS bridge.</p>"},{"location":"en/start/underlay/#create-vlan","title":"Create VLAN","text":"<p>Vlan provides an abstraction to bind Vlan Tag and ProviderNetwork.</p> <p>Create a VLAN as below:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Vlan\nmetadata:\n  name: vlan1\nspec:\n  id: 0\n  provider: net1\n</code></pre> <ul> <li><code>id</code>: VLAN ID/Tag\uff0cKube-OVN will add this Vlan tag to traffic, if set 0, no tag is added.</li> <li><code>provider</code>: The name of ProviderNetwork. Multiple VLAN can use a same ProviderNetwork.</li> </ul>"},{"location":"en/start/underlay/#create-subnet","title":"Create Subnet","text":"<p>Bind Vlan to a Subnet as below\uff1a</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: subnet1\nspec:\nprotocol: IPv4\ncidrBlock: 172.17.0.0/16\ngateway: 172.17.0.1\nvlan: vlan1\n</code></pre> <p>Simply specify the value of <code>vlan</code> as the name of the VLAN to be used. Multiple subnets can refer to the same VLAN.</p>"},{"location":"en/start/underlay/#create-pod","title":"Create Pod","text":"<p>You can create containers in the normal way, check whether the container IP is in the specified range and whether the container can interoperate with the physical network.</p> <p>For fixed IP requirements, please refer to Fixed Addresses</p>"},{"location":"en/start/underlay/#logical-gateway","title":"Logical Gateway","text":"<p>For cases where no gateway exists in the physical network, Kube-OVN supports the use of logical gateways configured in the subnet in Underlay mode. To use this feature, set <code>spec.logicalGateway</code> to <code>true</code> for the subnet:</p> <pre><code>apiVersion: kubeovn.io/v1\nkind: Subnet\nmetadata:\nname: subnet1\nspec:\nprotocol: IPv4\ncidrBlock: 172.17.0.0/16\ngateway: 172.17.0.1\nvlan: vlan1\nlogicalGateway: true\n</code></pre> <p>When this feature is turned on, the Pod does not use an external gateway, but a Logical Router created by Kube-OVN to forward cross-subnet communication.</p>"},{"location":"en/start/underlay/#interconnection-of-underlay-and-overlay-networks","title":"Interconnection of Underlay and Overlay Networks","text":"<p>If a cluster has both Underlay and Overlay subnets, by default, Pods in the Overlay subnet can access the Pod IPs in the Underlay subnet via a gateway using NAT. From the perspective of Pods in the Underlay subnet, the addresses in the Overlay subnet are external, and require the underlying physical device to forward, but the underlying physical device does not know the addresses in the Overlay subnet and cannot forward. Therefore, Pods in the Underlay subnet cannot access Pods in the Overlay subnet directly via Pod IPs.</p> <p>If you need to enable communication between Underlay and Overlay networks, you need to set the <code>u2oInterconnection</code> of the subnet to <code>true</code>. In this case, Kube-OVN will use an additional Underlay IP to connect the Underlay subnet and the <code>ovn-cluster</code> logical router, and set the corresponding routing rules to enable communication. Unlike the logical gateway, this solution only connects the Underlay and Overlay subnets within Kube-OVN, and other traffic accessing the Internet will still be forwarded through the physical gateway.</p>"},{"location":"en/start/underlay/#specify-logical-gateway-ip","title":"Specify logical gateway IP","text":"<p>After the interworking function is enabled, an IP from the subnet will be randomly selected as the logical gateway. If you need to specify the logical gateway of the Underlay Subnet, you can specify the field <code>u2oInterconnectionIP</code>.</p>"},{"location":"en/start/underlay/#specify-custom-vpc-for-underlay-subnet-connection","title":"Specify custom VPC for Underlay Subnet connection","text":"<p>By default, the Underlay Subnet will communicate with the Overlay Subnet on the default VPC. If you want to specify to communicate with a certain VPC, after setting <code>u2oInterconnection</code> to <code>true</code>, specify the <code>subnet.spec.vpc</code> field as the name of the VPC.</p>"},{"location":"en/start/underlay/#notice","title":"Notice","text":"<p>If you have an IP address configured on the network card of the node you are using, and the operating system configures the network using Netplan (such as Ubuntu), it is recommended that you set the renderer of Netplan to NetworkManager and configure a static IP address for the node's network card (disable DHCP).</p> <pre><code>network:\nrenderer: NetworkManager\nethernets:\neth0:\ndhcp4: no\naddresses:\n- 172.16.143.129/24\nversion: 2\n</code></pre> <p>If you want to modify the IP or routing configuration of the network card, you need to execute the following commands after modifying the Netplan configuration:</p> <pre><code>netplan generate\n\nnmcli connection reload netplan-eth0\nnmcli device set eth0 managed yes\n</code></pre> <p>After executing the above commands, Kube-OVN will transfer the IP and routing from the network card to the OVS bridge.</p> <p>If your operating system manages the network using NetworkManager (such as CentOS), you need to execute the following command after modifying the network card configuration:</p> <pre><code>nmcli connection reload eth0\nnmcli device set eth0 managed yes\nnmcli -t -f GENERAL.STATE device show eth0 | grep -qw unmanaged || nmcli device reapply eth0\n</code></pre> <p>Notice\uff1aIf the host nic's MAC is changed, Kube-OVN will not change the OVS bridge's MAC unless kube-ovn-cni is restarted.</p>"},{"location":"en/start/underlay/#known-issues","title":"Known Issues","text":""},{"location":"en/start/underlay/#when-the-physical-network-is-enabled-with-hairpin-pod-network-is-abnormal","title":"When the physical network is enabled with hairpin, Pod network is abnormal","text":"<p>When physical networks enable hairpin or similar behaviors, problems such as gateway check failure when creating Pods and abnormal network communication of Pods may occur. This is because the default MAC learning function of OVS bridge does not support this kind of network environment.</p> <p>To solve this problem, it is necessary to turn off hairpin (or modify the relevant configuration of physical network), or update the Kube-OVN version.</p>"},{"location":"en/start/underlay/#when-there-are-a-large-number-of-pods-gateway-check-for-new-pods-fails","title":"When there are a large number of Pods, gateway check for new Pods fails","text":"<p>If there are a large number of Pods running on the same node (more than 300), it may cause packet loss due to the OVS flow table resubmit times exceeding the upper limit of ARP broadcast packets.</p> <pre><code>2022-11-13T08:43:46.782Z|00222|ofproto_dpif_upcall(handler5)|WARN|Flow: arp,in_port=331,vlan_tci=0x0000,dl_src=00:00:00:25:eb:39,dl_dst=ff:ff:ff:ff:ff:ff,arp_spa=10.213.131.240,arp_tpa=10.213.159.254,arp_op=1,arp_sha=00:00:00:25:eb:39,arp_tha=ff:ff:ff:ff:ff:ff\n\nbridge(\"br-int\")\n----------------\n 0. No match.\n     &gt;&gt;&gt;&gt; received packet on unknown port 331 &lt;&lt;&lt;&lt;\n    drop\n\nFinal flow: unchanged\nMegaflow: recirc_id=0,eth,arp,in_port=331,dl_src=00:00:00:25:eb:39\nDatapath actions: drop\n2022-11-13T08:44:34.077Z|00224|ofproto_dpif_xlate(handler5)|WARN|over 4096 resubmit actions on bridge br-int while processing arp,in_port=13483,vlan_tci=0x0000,dl_src=00:00:00:59:ef:13,dl_dst=ff:ff:ff:ff:ff:ff,arp_spa=10.213.152.3,arp_tpa=10.213.159.254,arp_op=1,arp_sha=00:00:00:59:ef:13,arp_tha=ff:ff:ff:ff:ff:ff\n</code></pre> <p>To solve this issue, modify the OVN NB option <code>bcast_arp_req_flood</code> to <code>false</code>:</p> <pre><code>kubectl ko nbctl set NB_Global . options:bcast_arp_req_flood=false\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"},{"location":"en/start/uninstall/","title":"Uninstall","text":"<p>If you need to remove the Kube-OVN and replace it with another network plugin, please follow the steps below to remove all the corresponding Kube-OVN component and OVS configuration to avoid interference with other network plugins.</p> <p>Feel free to contact us with an Issue to give us feedback on why you don't use Kube-OVN to help us improve it.</p>"},{"location":"en/start/uninstall/#delete-resource-in-kubernetes","title":"Delete Resource in Kubernetes","text":"<p>Download and run the script below to delete resource created in Kubernetes:</p> <pre><code>wget https://raw.githubusercontent.com/kubeovn/kube-ovn/master/dist/images/cleanup.sh\nbash cleanup.sh\n</code></pre>"},{"location":"en/start/uninstall/#cleanup-config-and-logs-on-every-node","title":"Cleanup Config and Logs on Every Node","text":"<p>Run the following commands on each node to clean up the configuration retained by ovsdb and openvswitch:</p> <pre><code>rm -rf /var/run/openvswitch\nrm -rf /var/run/ovn\nrm -rf /etc/origin/openvswitch/\nrm -rf /etc/origin/ovn/\nrm -rf /etc/cni/net.d/00-kube-ovn.conflist\nrm -rf /etc/cni/net.d/01-kube-ovn.conflist\nrm -rf /var/log/openvswitch\nrm -rf /var/log/ovn\nrm -fr /var/log/kube-ovn\n</code></pre>"},{"location":"en/start/uninstall/#reboot-node","title":"Reboot Node","text":"<p>Reboot the machine to ensure that the corresponding NIC information and iptable/ipset rules are cleared to avoid the interference with other network plugins:</p> <pre><code>reboot\n</code></pre> <p> \u5fae\u4fe1\u7fa4  Slack  Twitter  Support</p>"}]}
\ No newline at end of file
diff --git a/v1.13.x/sitemap.xml b/v1.13.x/sitemap.xml
index bba0af796..e8bd1ae8d 100644
--- a/v1.13.x/sitemap.xml
+++ b/v1.13.x/sitemap.xml
@@ -2,7 +2,7 @@
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml">
     <url>
          <loc>None./</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/"/>
@@ -10,7 +10,7 @@
     </url>
     <url>
          <loc>Nonecontact/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/contact/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/contact/"/>
@@ -18,7 +18,7 @@
     </url>
     <url>
          <loc>Noneadvance/accelerate-intra-node-tcp-with-ebpf/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/accelerate-intra-node-tcp-with-ebpf/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/accelerate-intra-node-tcp-with-ebpf/"/>
@@ -26,7 +26,7 @@
     </url>
     <url>
          <loc>Noneadvance/cilium-hubble-observe/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/cilium-hubble-observe/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/cilium-hubble-observe/"/>
@@ -34,7 +34,7 @@
     </url>
     <url>
          <loc>Noneadvance/cilium-networkpolicy/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/cilium-networkpolicy/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/cilium-networkpolicy/"/>
@@ -42,7 +42,7 @@
     </url>
     <url>
          <loc>Noneadvance/dhcp/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/dhcp/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/dhcp/"/>
@@ -50,7 +50,7 @@
     </url>
     <url>
          <loc>Noneadvance/dpdk/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/dpdk/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/dpdk/"/>
@@ -58,7 +58,7 @@
     </url>
     <url>
          <loc>Noneadvance/external-gateway/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/external-gateway/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/external-gateway/"/>
@@ -66,7 +66,7 @@
     </url>
     <url>
          <loc>Noneadvance/fastpath/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/fastpath/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/fastpath/"/>
@@ -74,7 +74,7 @@
     </url>
     <url>
          <loc>Noneadvance/multi-nic/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/multi-nic/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/multi-nic/"/>
@@ -82,7 +82,7 @@
     </url>
     <url>
          <loc>Noneadvance/nat-policy-rule/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/nat-policy-rule/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/nat-policy-rule/"/>
@@ -90,7 +90,7 @@
     </url>
     <url>
          <loc>Noneadvance/node-local-dns/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/node-local-dns/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/node-local-dns/"/>
@@ -98,7 +98,7 @@
     </url>
     <url>
          <loc>Noneadvance/offload-corigine/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/offload-corigine/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/offload-corigine/"/>
@@ -106,7 +106,7 @@
     </url>
     <url>
          <loc>Noneadvance/offload-mellanox/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/offload-mellanox/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/offload-mellanox/"/>
@@ -114,7 +114,7 @@
     </url>
     <url>
          <loc>Noneadvance/overlay-with-route/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/overlay-with-route/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/overlay-with-route/"/>
@@ -122,7 +122,7 @@
     </url>
     <url>
          <loc>Noneadvance/ovn-eip-fip-snat/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/ovn-eip-fip-snat/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/ovn-eip-fip-snat/"/>
@@ -130,7 +130,7 @@
     </url>
     <url>
          <loc>Noneadvance/ovn-ipsec/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/ovn-ipsec/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/ovn-ipsec/"/>
@@ -138,7 +138,7 @@
     </url>
     <url>
          <loc>Noneadvance/ovn-l3-ha-based-ecmp-with-bfd/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/ovn-l3-ha-based-ecmp-with-bfd/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/ovn-l3-ha-based-ecmp-with-bfd/"/>
@@ -146,7 +146,7 @@
     </url>
     <url>
          <loc>Noneadvance/ovn-remote-port-mirroring/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/ovn-remote-port-mirroring/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/ovn-remote-port-mirroring/"/>
@@ -154,7 +154,7 @@
     </url>
     <url>
          <loc>Noneadvance/performance-tuning/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/performance-tuning/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/performance-tuning/"/>
@@ -162,7 +162,7 @@
     </url>
     <url>
          <loc>Noneadvance/security-group/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/security-group/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/security-group/"/>
@@ -170,7 +170,7 @@
     </url>
     <url>
          <loc>Noneadvance/vip/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/vip/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/vip/"/>
@@ -178,7 +178,7 @@
     </url>
     <url>
          <loc>Noneadvance/vpc-dns/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/vpc-dns/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/vpc-dns/"/>
@@ -186,7 +186,7 @@
     </url>
     <url>
          <loc>Noneadvance/vpc-internal-lb/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/vpc-internal-lb/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/vpc-internal-lb/"/>
@@ -194,7 +194,7 @@
     </url>
     <url>
          <loc>Noneadvance/vpc-peering/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/vpc-peering/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/vpc-peering/"/>
@@ -202,7 +202,7 @@
     </url>
     <url>
          <loc>Noneadvance/windows/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/windows/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/windows/"/>
@@ -210,7 +210,7 @@
     </url>
     <url>
          <loc>Noneadvance/with-bgp/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/with-bgp/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/with-bgp/"/>
@@ -218,7 +218,7 @@
     </url>
     <url>
          <loc>Noneadvance/with-cilium/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/with-cilium/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/with-cilium/"/>
@@ -226,7 +226,7 @@
     </url>
     <url>
          <loc>Noneadvance/with-openstack/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/with-openstack/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/with-openstack/"/>
@@ -234,7 +234,7 @@
     </url>
     <url>
          <loc>Noneadvance/with-ovn-ic/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/with-ovn-ic/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/with-ovn-ic/"/>
@@ -242,7 +242,7 @@
     </url>
     <url>
          <loc>Noneadvance/with-submariner/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/advance/with-submariner/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/advance/with-submariner/"/>
@@ -250,7 +250,7 @@
     </url>
     <url>
          <loc>Noneguide/custom-routes/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/guide/custom-routes/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/guide/custom-routes/"/>
@@ -258,7 +258,7 @@
     </url>
     <url>
          <loc>Noneguide/dual-stack/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/guide/dual-stack/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/guide/dual-stack/"/>
@@ -266,7 +266,7 @@
     </url>
     <url>
          <loc>Noneguide/eip-snat/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/guide/eip-snat/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/guide/eip-snat/"/>
@@ -274,7 +274,7 @@
     </url>
     <url>
          <loc>Noneguide/ippool/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/guide/ippool/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/guide/ippool/"/>
@@ -282,7 +282,7 @@
     </url>
     <url>
          <loc>Noneguide/loadbalancer-service/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/guide/loadbalancer-service/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/guide/loadbalancer-service/"/>
@@ -290,7 +290,7 @@
     </url>
     <url>
          <loc>Noneguide/mirror/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/guide/mirror/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/guide/mirror/"/>
@@ -298,7 +298,7 @@
     </url>
     <url>
          <loc>Noneguide/networkpolicy-log/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/guide/networkpolicy-log/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/guide/networkpolicy-log/"/>
@@ -306,7 +306,7 @@
     </url>
     <url>
          <loc>Noneguide/prometheus-grafana/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/guide/prometheus-grafana/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/guide/prometheus-grafana/"/>
@@ -314,7 +314,7 @@
     </url>
     <url>
          <loc>Noneguide/prometheus/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/guide/prometheus/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/guide/prometheus/"/>
@@ -322,7 +322,7 @@
     </url>
     <url>
          <loc>Noneguide/qos/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/guide/qos/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/guide/qos/"/>
@@ -330,7 +330,7 @@
     </url>
     <url>
          <loc>Noneguide/setup-options/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/guide/setup-options/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/guide/setup-options/"/>
@@ -338,7 +338,7 @@
     </url>
     <url>
          <loc>Noneguide/static-ip-mac/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/guide/static-ip-mac/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/guide/static-ip-mac/"/>
@@ -346,7 +346,7 @@
     </url>
     <url>
          <loc>Noneguide/subnet/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/guide/subnet/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/guide/subnet/"/>
@@ -354,7 +354,7 @@
     </url>
     <url>
          <loc>Noneguide/vpc-qos/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/guide/vpc-qos/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/guide/vpc-qos/"/>
@@ -362,7 +362,7 @@
     </url>
     <url>
          <loc>Noneguide/vpc/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/guide/vpc/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/guide/vpc/"/>
@@ -370,7 +370,7 @@
     </url>
     <url>
          <loc>Noneguide/webhook/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/guide/webhook/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/guide/webhook/"/>
@@ -378,7 +378,7 @@
     </url>
     <url>
          <loc>Noneops/change-default-subnet/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/ops/change-default-subnet/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/ops/change-default-subnet/"/>
@@ -386,7 +386,7 @@
     </url>
     <url>
          <loc>Noneops/change-join-subnet/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/ops/change-join-subnet/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/ops/change-join-subnet/"/>
@@ -394,7 +394,7 @@
     </url>
     <url>
          <loc>Noneops/change-log-level/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/ops/change-log-level/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/ops/change-log-level/"/>
@@ -402,7 +402,7 @@
     </url>
     <url>
          <loc>Noneops/change-ovn-central-node/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/ops/change-ovn-central-node/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/ops/change-ovn-central-node/"/>
@@ -410,7 +410,7 @@
     </url>
     <url>
          <loc>Noneops/delete-worker-node/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/ops/delete-worker-node/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/ops/delete-worker-node/"/>
@@ -418,7 +418,7 @@
     </url>
     <url>
          <loc>Noneops/faq/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/ops/faq/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/ops/faq/"/>
@@ -426,7 +426,7 @@
     </url>
     <url>
          <loc>Noneops/from-calico/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/ops/from-calico/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/ops/from-calico/"/>
@@ -434,7 +434,7 @@
     </url>
     <url>
          <loc>Noneops/kubectl-ko/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/ops/kubectl-ko/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/ops/kubectl-ko/"/>
@@ -442,7 +442,7 @@
     </url>
     <url>
          <loc>Noneops/recover-db/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/ops/recover-db/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/ops/recover-db/"/>
@@ -450,7 +450,7 @@
     </url>
     <url>
          <loc>Nonereference/architecture/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/reference/architecture/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/reference/architecture/"/>
@@ -458,7 +458,7 @@
     </url>
     <url>
          <loc>Nonereference/dev-env/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/reference/dev-env/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/reference/dev-env/"/>
@@ -466,7 +466,7 @@
     </url>
     <url>
          <loc>Nonereference/document-convention/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/reference/document-convention/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/reference/document-convention/"/>
@@ -474,7 +474,7 @@
     </url>
     <url>
          <loc>Nonereference/feature-stage/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/reference/feature-stage/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/reference/feature-stage/"/>
@@ -482,7 +482,7 @@
     </url>
     <url>
          <loc>Nonereference/iptables-rules/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/reference/iptables-rules/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/reference/iptables-rules/"/>
@@ -490,7 +490,7 @@
     </url>
     <url>
          <loc>Nonereference/kube-ovn-api/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/reference/kube-ovn-api/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/reference/kube-ovn-api/"/>
@@ -498,7 +498,7 @@
     </url>
     <url>
          <loc>Nonereference/kube-ovn-pinger-args/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/reference/kube-ovn-pinger-args/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/reference/kube-ovn-pinger-args/"/>
@@ -506,7 +506,7 @@
     </url>
     <url>
          <loc>Nonereference/metrics/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/reference/metrics/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/reference/metrics/"/>
@@ -514,7 +514,7 @@
     </url>
     <url>
          <loc>Nonereference/ovs-ovn-customized/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/reference/ovs-ovn-customized/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/reference/ovs-ovn-customized/"/>
@@ -522,7 +522,7 @@
     </url>
     <url>
          <loc>Nonereference/tunnel-protocol/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/reference/tunnel-protocol/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/reference/tunnel-protocol/"/>
@@ -530,7 +530,7 @@
     </url>
     <url>
          <loc>Nonereference/underlay-topology/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/reference/underlay-topology/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/reference/underlay-topology/"/>
@@ -538,7 +538,7 @@
     </url>
     <url>
          <loc>Nonestart/one-step-install/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/start/one-step-install/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/start/one-step-install/"/>
@@ -546,7 +546,7 @@
     </url>
     <url>
          <loc>Nonestart/prepare/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/start/prepare/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/start/prepare/"/>
@@ -554,7 +554,7 @@
     </url>
     <url>
          <loc>Nonestart/sealos-install/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/start/sealos-install/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/start/sealos-install/"/>
@@ -562,7 +562,7 @@
     </url>
     <url>
          <loc>Nonestart/underlay/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/start/underlay/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/start/underlay/"/>
@@ -570,7 +570,7 @@
     </url>
     <url>
          <loc>Nonestart/uninstall/</loc>
-         <lastmod>2023-09-19</lastmod>
+         <lastmod>2023-09-26</lastmod>
          <changefreq>daily</changefreq>
          <xhtml:link rel="alternate" hreflang="en" href="Noneen/start/uninstall/"/>
          <xhtml:link rel="alternate" hreflang="zh" href="Nonezh/start/uninstall/"/>
diff --git a/v1.13.x/sitemap.xml.gz b/v1.13.x/sitemap.xml.gz
index 4068e5435..8d2bab6d1 100644
Binary files a/v1.13.x/sitemap.xml.gz and b/v1.13.x/sitemap.xml.gz differ
diff --git a/v1.13.x/zh/guide/custom-routes/index.html b/v1.13.x/zh/guide/custom-routes/index.html
index c7f86157d..430b3e132 100644
--- a/v1.13.x/zh/guide/custom-routes/index.html
+++ b/v1.13.x/zh/guide/custom-routes/index.html
@@ -17,7 +17,7 @@
 <span class=w>        </span><span class=no>&quot;gw&quot;: &quot;10.16.0.254&quot;</span>
 <span class=w>      </span><span class=no>}, {</span>
 <span class=w>        </span><span class=no>&quot;gw&quot;: &quot;10.16.0.254&quot;</span>
-<span class=w>      </span><span class=no>}]&#39;</span>
+<span class=w>      </span><span class=no>}]</span>
 <span class=nt>spec</span><span class=p>:</span>
 <span class=w>  </span><span class=nt>containers</span><span class=p>:</span>
 <span class=w>  </span><span class="p p-Indicator">-</span><span class=w> </span><span class=nt>name</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">nginx</span>
@@ -44,12 +44,12 @@
 <span class=w>            </span><span class=no>&quot;gw&quot;: &quot;10.16.0.254&quot;</span>
 <span class=w>          </span><span class=no>}, {</span>
 <span class=w>            </span><span class=no>&quot;gw&quot;: &quot;10.16.0.254&quot;</span>
-<span class=w>          </span><span class=no>}]&#39;</span>
+<span class=w>          </span><span class=no>}]</span>
 <span class=w>    </span><span class=nt>spec</span><span class=p>:</span>
 <span class=w>      </span><span class=nt>containers</span><span class=p>:</span>
 <span class=w>      </span><span class="p p-Indicator">-</span><span class=w> </span><span class=nt>name</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">nginx</span>
 <span class=w>        </span><span class=nt>image</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">docker.io/library/nginx:alpine</span>
-</code></pre></div> <p><a class=md-button href=https://ma.alauda.cn/p/0f39a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M9.5 4C5.36 4 2 6.69 2 10c0 1.89 1.08 3.56 2.78 4.66L4 17l2.5-1.5c.89.31 1.87.5 2.91.5A5.22 5.22 0 0 1 9 14c0-3.31 3.13-6 7-6 .19 0 .38 0 .56.03C15.54 5.69 12.78 4 9.5 4m-3 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m5 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1M16 9c-3.31 0-6 2.24-6 5s2.69 5 6 5c.67 0 1.31-.08 1.91-.25L20 20l-.62-1.87C20.95 17.22 22 15.71 22 14c0-2.76-2.69-5-6-5m-2 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m4 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1Z"/></svg></span> 微信群</a> <a class=md-button href=https://communityinviter.com/apps/kube-ovn/kube-ovn/ target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M6 15a2 2 0 0 1-2 2 2 2 0 0 1-2-2 2 2 0 0 1 2-2h2v2m1 0a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-5m2-8a2 2 0 0 1-2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2v2H9m0 1a2 2 0 0 1 2 2 2 2 0 0 1-2 2H4a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5m8 2a2 2 0 0 1 2-2 2 2 0 0 1 2 2 2 2 0 0 1-2 2h-2v-2m-1 0a2 2 0 0 1-2 2 2 2 0 0 1-2-2V5a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5m-2 8a2 2 0 0 1 2 2 2 2 0 0 1-2 2 2 2 0 0 1-2-2v-2h2m0-1a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-5Z"/></svg></span> Slack</a> <a class=md-button href=https://twitter.com/KubeOvn target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M22.46 6c-.77.35-1.6.58-2.46.69.88-.53 1.56-1.37 1.88-2.38-.83.5-1.75.85-2.72 1.05C18.37 4.5 17.26 4 16 4c-2.35 0-4.27 1.92-4.27 4.29 0 .34.04.67.11.98C8.28 9.09 5.11 7.38 3 4.79c-.37.63-.58 1.37-.58 2.15 0 1.49.75 2.81 1.91 3.56-.71 0-1.37-.2-1.95-.5v.03c0 2.08 1.48 3.82 3.44 4.21a4.22 4.22 0 0 1-1.93.07 4.28 4.28 0 0 0 4 2.98 8.521 8.521 0 0 1-5.33 1.84c-.34 0-.68-.02-1.02-.06C3.44 20.29 5.7 21 8.12 21 16 21 20.33 14.46 20.33 8.79c0-.19 0-.37-.01-.56.84-.6 1.56-1.36 2.14-2.23Z"/></svg></span> Twitter</a> <a class=md-button href=https://ma.alauda.cn/p/2f53a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="m20 8-8 5-8-5V6l8 5 8-5m0-2H4c-1.11 0-2 .89-2 2v12a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V6a2 2 0 0 0-2-2Z"/></svg></span> Support</a></p> <hr> <div class=md-source-file> <small> 最后更新: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2023年5月9日</span> <br> 创建日期: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2023年2月16日</span> </small> </div> <h2 id=__comments>评论</h2> <script src=https://giscus.app/client.js data-repo=kubeovn/kube-ovn data-repo-id="MDEwOlJlcG9zaXRvcnkxNzcwNjg5NjE=" data-category=Announcements data-category-id=DIC_kwDOCo3boc4CAj60 data-mapping=pathname data-reactions-enabled=1 data-emit-metadata=0 data-input-position=bottom data-theme=light data-lang=zh-CN crossorigin=anonymous async>
+</code></pre></div> <p><a class=md-button href=https://ma.alauda.cn/p/0f39a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M9.5 4C5.36 4 2 6.69 2 10c0 1.89 1.08 3.56 2.78 4.66L4 17l2.5-1.5c.89.31 1.87.5 2.91.5A5.22 5.22 0 0 1 9 14c0-3.31 3.13-6 7-6 .19 0 .38 0 .56.03C15.54 5.69 12.78 4 9.5 4m-3 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m5 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1M16 9c-3.31 0-6 2.24-6 5s2.69 5 6 5c.67 0 1.31-.08 1.91-.25L20 20l-.62-1.87C20.95 17.22 22 15.71 22 14c0-2.76-2.69-5-6-5m-2 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m4 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1Z"/></svg></span> 微信群</a> <a class=md-button href=https://communityinviter.com/apps/kube-ovn/kube-ovn/ target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M6 15a2 2 0 0 1-2 2 2 2 0 0 1-2-2 2 2 0 0 1 2-2h2v2m1 0a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-5m2-8a2 2 0 0 1-2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2v2H9m0 1a2 2 0 0 1 2 2 2 2 0 0 1-2 2H4a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5m8 2a2 2 0 0 1 2-2 2 2 0 0 1 2 2 2 2 0 0 1-2 2h-2v-2m-1 0a2 2 0 0 1-2 2 2 2 0 0 1-2-2V5a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5m-2 8a2 2 0 0 1 2 2 2 2 0 0 1-2 2 2 2 0 0 1-2-2v-2h2m0-1a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-5Z"/></svg></span> Slack</a> <a class=md-button href=https://twitter.com/KubeOvn target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M22.46 6c-.77.35-1.6.58-2.46.69.88-.53 1.56-1.37 1.88-2.38-.83.5-1.75.85-2.72 1.05C18.37 4.5 17.26 4 16 4c-2.35 0-4.27 1.92-4.27 4.29 0 .34.04.67.11.98C8.28 9.09 5.11 7.38 3 4.79c-.37.63-.58 1.37-.58 2.15 0 1.49.75 2.81 1.91 3.56-.71 0-1.37-.2-1.95-.5v.03c0 2.08 1.48 3.82 3.44 4.21a4.22 4.22 0 0 1-1.93.07 4.28 4.28 0 0 0 4 2.98 8.521 8.521 0 0 1-5.33 1.84c-.34 0-.68-.02-1.02-.06C3.44 20.29 5.7 21 8.12 21 16 21 20.33 14.46 20.33 8.79c0-.19 0-.37-.01-.56.84-.6 1.56-1.36 2.14-2.23Z"/></svg></span> Twitter</a> <a class=md-button href=https://ma.alauda.cn/p/2f53a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="m20 8-8 5-8-5V6l8 5 8-5m0-2H4c-1.11 0-2 .89-2 2v12a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V6a2 2 0 0 0-2-2Z"/></svg></span> Support</a></p> <hr> <div class=md-source-file> <small> 最后更新: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2023年9月26日</span> <br> 创建日期: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2023年2月16日</span> </small> </div> <h2 id=__comments>评论</h2> <script src=https://giscus.app/client.js data-repo=kubeovn/kube-ovn data-repo-id="MDEwOlJlcG9zaXRvcnkxNzcwNjg5NjE=" data-category=Announcements data-category-id=DIC_kwDOCo3boc4CAj60 data-mapping=pathname data-reactions-enabled=1 data-emit-metadata=0 data-input-position=bottom data-theme=light data-lang=zh-CN crossorigin=anonymous async>
 </script> <script>
     var palette = __md_get("__palette")
     if (palette && typeof palette.color === "object")
diff --git a/v1.13.x/zh/guide/static-ip-mac/index.html b/v1.13.x/zh/guide/static-ip-mac/index.html
index 592a76ad8..18c2cf9ed 100644
--- a/v1.13.x/zh/guide/static-ip-mac/index.html
+++ b/v1.13.x/zh/guide/static-ip-mac/index.html
@@ -17,7 +17,7 @@
 <span class=w>  </span><span class=nt>containers</span><span class=p>:</span>
 <span class=w>  </span><span class="p p-Indicator">-</span><span class=w> </span><span class=nt>name</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">static-ip</span>
 <span class=w>    </span><span class=nt>image</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">docker.io/library/nginx:alpine</span>
-</code></pre></div> <p>在使用 annotation 定义单个 Pod IP/Mac 时需要注意以下几点：</p> <ol> <li>所使用的 IP/Mac 不能和已有的 IP/Mac 冲突。</li> <li>IP 必须在所属子网的 CIDR 内。</li> <li>可以只指定 IP 或 Mac，只指定一个时，另一个会随机分配。</li> </ol> <h2 id=workload-ip-pool>Workload 通用 IP Pool 固定地址<a class=headerlink href=#workload-ip-pool title="Permanent link">&para;</a></h2> <p>Kube-OVN 支持通过 annotation <code>ovn.kubernetes.io/ip_pool</code> 给 Workload（Deployment/StatefulSet/DaemonSet/Job/CronJob）设置固定 IP。 <code>kube-ovn-controllerr</code> 会自动选择 <code>ovn.kubernetes.io/ip_pool</code> 中指定的 IP 并进行冲突检测。</p> <p>IP Pool 的 Annotation 需要加在 <code>template</code> 内的 <code>annotation</code> 字段，除了 Kubernetes 内置的 Workload 类型， 其他用户自定义的 Workload 也可以使用同样的方式进行固定地址分配。</p> <h3 id=deployment-ip>Deployment 固定 IP 示例<a class=headerlink href=#deployment-ip title="Permanent link">&para;</a></h3> <div class=highlight><pre><span></span><code><span class=nt>apiVersion</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">apps/v1</span>
+</code></pre></div> <p>在使用 annotation 定义单个 Pod IP/Mac 时需要注意以下几点：</p> <ol> <li>所使用的 IP/Mac 不能和已有的 IP/Mac 冲突。</li> <li>IP 必须在所属子网的 CIDR 内。</li> <li>可以只指定 IP 或 Mac，只指定一个时，另一个会随机分配。</li> </ol> <h2 id=workload-ip-pool>Workload 通用 IP Pool 固定地址<a class=headerlink href=#workload-ip-pool title="Permanent link">&para;</a></h2> <p>Kube-OVN 支持通过 annotation <code>ovn.kubernetes.io/ip_pool</code> 给 Workload（Deployment/StatefulSet/DaemonSet/Job/CronJob）设置固定 IP。 <code>kube-ovn-controller</code> 会自动选择 <code>ovn.kubernetes.io/ip_pool</code> 中指定的 IP 并进行冲突检测。</p> <p>IP Pool 的 Annotation 需要加在 <code>template</code> 内的 <code>annotation</code> 字段，除了 Kubernetes 内置的 Workload 类型， 其他用户自定义的 Workload 也可以使用同样的方式进行固定地址分配。</p> <h3 id=deployment-ip>Deployment 固定 IP 示例<a class=headerlink href=#deployment-ip title="Permanent link">&para;</a></h3> <div class=highlight><pre><span></span><code><span class=nt>apiVersion</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">apps/v1</span>
 <span class=nt>kind</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">Deployment</span>
 <span class=nt>metadata</span><span class=p>:</span>
 <span class=w>  </span><span class=nt>name</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">ippool</span>
@@ -59,7 +59,7 @@
 <span class=w>        </span><span class=nt>ports</span><span class=p>:</span>
 <span class=w>        </span><span class="p p-Indicator">-</span><span class=w> </span><span class=nt>containerPort</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">80</span>
 <span class=w>          </span><span class=nt>name</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">web</span>
-</code></pre></div> <p>可以尝试删除 StatefulSet 下 Pod 观察 Pod IP 变化信息。</p> <h2 id=kubevirt-vm>KubeVirt VM 固定地址<a class=headerlink href=#kubevirt-vm title="Permanent link">&para;</a></h2> <p>针对 KubeVirt 创建的 VM 实例，<code>kube-ovn-controller</code> 可以按照类似 StatefulSet Pod 的方式进行 IP 地址分配和管理。 以达到 VM 实例在生命周期内启停，升级，迁移等操作过程中地址固定不变，更符虚拟化合用户的实际使用体验。</p> <p><a class=md-button href=https://ma.alauda.cn/p/0f39a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M9.5 4C5.36 4 2 6.69 2 10c0 1.89 1.08 3.56 2.78 4.66L4 17l2.5-1.5c.89.31 1.87.5 2.91.5A5.22 5.22 0 0 1 9 14c0-3.31 3.13-6 7-6 .19 0 .38 0 .56.03C15.54 5.69 12.78 4 9.5 4m-3 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m5 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1M16 9c-3.31 0-6 2.24-6 5s2.69 5 6 5c.67 0 1.31-.08 1.91-.25L20 20l-.62-1.87C20.95 17.22 22 15.71 22 14c0-2.76-2.69-5-6-5m-2 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m4 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1Z"/></svg></span> 微信群</a> <a class=md-button href=https://communityinviter.com/apps/kube-ovn/kube-ovn/ target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M6 15a2 2 0 0 1-2 2 2 2 0 0 1-2-2 2 2 0 0 1 2-2h2v2m1 0a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-5m2-8a2 2 0 0 1-2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2v2H9m0 1a2 2 0 0 1 2 2 2 2 0 0 1-2 2H4a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5m8 2a2 2 0 0 1 2-2 2 2 0 0 1 2 2 2 2 0 0 1-2 2h-2v-2m-1 0a2 2 0 0 1-2 2 2 2 0 0 1-2-2V5a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5m-2 8a2 2 0 0 1 2 2 2 2 0 0 1-2 2 2 2 0 0 1-2-2v-2h2m0-1a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-5Z"/></svg></span> Slack</a> <a class=md-button href=https://twitter.com/KubeOvn target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M22.46 6c-.77.35-1.6.58-2.46.69.88-.53 1.56-1.37 1.88-2.38-.83.5-1.75.85-2.72 1.05C18.37 4.5 17.26 4 16 4c-2.35 0-4.27 1.92-4.27 4.29 0 .34.04.67.11.98C8.28 9.09 5.11 7.38 3 4.79c-.37.63-.58 1.37-.58 2.15 0 1.49.75 2.81 1.91 3.56-.71 0-1.37-.2-1.95-.5v.03c0 2.08 1.48 3.82 3.44 4.21a4.22 4.22 0 0 1-1.93.07 4.28 4.28 0 0 0 4 2.98 8.521 8.521 0 0 1-5.33 1.84c-.34 0-.68-.02-1.02-.06C3.44 20.29 5.7 21 8.12 21 16 21 20.33 14.46 20.33 8.79c0-.19 0-.37-.01-.56.84-.6 1.56-1.36 2.14-2.23Z"/></svg></span> Twitter</a> <a class=md-button href=https://ma.alauda.cn/p/2f53a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="m20 8-8 5-8-5V6l8 5 8-5m0-2H4c-1.11 0-2 .89-2 2v12a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V6a2 2 0 0 0-2-2Z"/></svg></span> Support</a></p> <hr> <div class=md-source-file> <small> 最后更新: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2023年9月12日</span> <br> 创建日期: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2022年5月20日</span> </small> </div> <h2 id=__comments>评论</h2> <script src=https://giscus.app/client.js data-repo=kubeovn/kube-ovn data-repo-id="MDEwOlJlcG9zaXRvcnkxNzcwNjg5NjE=" data-category=Announcements data-category-id=DIC_kwDOCo3boc4CAj60 data-mapping=pathname data-reactions-enabled=1 data-emit-metadata=0 data-input-position=bottom data-theme=light data-lang=zh-CN crossorigin=anonymous async>
+</code></pre></div> <p>可以尝试删除 StatefulSet 下 Pod 观察 Pod IP 变化信息。</p> <h2 id=kubevirt-vm>KubeVirt VM 固定地址<a class=headerlink href=#kubevirt-vm title="Permanent link">&para;</a></h2> <p>针对 KubeVirt 创建的 VM 实例，<code>kube-ovn-controller</code> 可以按照类似 StatefulSet Pod 的方式进行 IP 地址分配和管理。 以达到 VM 实例在生命周期内启停，升级，迁移等操作过程中地址固定不变，更符虚拟化合用户的实际使用体验。</p> <p><a class=md-button href=https://ma.alauda.cn/p/0f39a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M9.5 4C5.36 4 2 6.69 2 10c0 1.89 1.08 3.56 2.78 4.66L4 17l2.5-1.5c.89.31 1.87.5 2.91.5A5.22 5.22 0 0 1 9 14c0-3.31 3.13-6 7-6 .19 0 .38 0 .56.03C15.54 5.69 12.78 4 9.5 4m-3 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m5 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1M16 9c-3.31 0-6 2.24-6 5s2.69 5 6 5c.67 0 1.31-.08 1.91-.25L20 20l-.62-1.87C20.95 17.22 22 15.71 22 14c0-2.76-2.69-5-6-5m-2 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m4 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1Z"/></svg></span> 微信群</a> <a class=md-button href=https://communityinviter.com/apps/kube-ovn/kube-ovn/ target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M6 15a2 2 0 0 1-2 2 2 2 0 0 1-2-2 2 2 0 0 1 2-2h2v2m1 0a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-5m2-8a2 2 0 0 1-2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2v2H9m0 1a2 2 0 0 1 2 2 2 2 0 0 1-2 2H4a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5m8 2a2 2 0 0 1 2-2 2 2 0 0 1 2 2 2 2 0 0 1-2 2h-2v-2m-1 0a2 2 0 0 1-2 2 2 2 0 0 1-2-2V5a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5m-2 8a2 2 0 0 1 2 2 2 2 0 0 1-2 2 2 2 0 0 1-2-2v-2h2m0-1a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-5Z"/></svg></span> Slack</a> <a class=md-button href=https://twitter.com/KubeOvn target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M22.46 6c-.77.35-1.6.58-2.46.69.88-.53 1.56-1.37 1.88-2.38-.83.5-1.75.85-2.72 1.05C18.37 4.5 17.26 4 16 4c-2.35 0-4.27 1.92-4.27 4.29 0 .34.04.67.11.98C8.28 9.09 5.11 7.38 3 4.79c-.37.63-.58 1.37-.58 2.15 0 1.49.75 2.81 1.91 3.56-.71 0-1.37-.2-1.95-.5v.03c0 2.08 1.48 3.82 3.44 4.21a4.22 4.22 0 0 1-1.93.07 4.28 4.28 0 0 0 4 2.98 8.521 8.521 0 0 1-5.33 1.84c-.34 0-.68-.02-1.02-.06C3.44 20.29 5.7 21 8.12 21 16 21 20.33 14.46 20.33 8.79c0-.19 0-.37-.01-.56.84-.6 1.56-1.36 2.14-2.23Z"/></svg></span> Twitter</a> <a class=md-button href=https://ma.alauda.cn/p/2f53a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="m20 8-8 5-8-5V6l8 5 8-5m0-2H4c-1.11 0-2 .89-2 2v12a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V6a2 2 0 0 0-2-2Z"/></svg></span> Support</a></p> <hr> <div class=md-source-file> <small> 最后更新: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2023年9月26日</span> <br> 创建日期: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2022年5月20日</span> </small> </div> <h2 id=__comments>评论</h2> <script src=https://giscus.app/client.js data-repo=kubeovn/kube-ovn data-repo-id="MDEwOlJlcG9zaXRvcnkxNzcwNjg5NjE=" data-category=Announcements data-category-id=DIC_kwDOCo3boc4CAj60 data-mapping=pathname data-reactions-enabled=1 data-emit-metadata=0 data-input-position=bottom data-theme=light data-lang=zh-CN crossorigin=anonymous async>
 </script> <script>
     var palette = __md_get("__palette")
     if (palette && typeof palette.color === "object")
diff --git a/v1.13.x/zh/guide/subnet/index.html b/v1.13.x/zh/guide/subnet/index.html
index 4f539b271..43fcfda4a 100644
--- a/v1.13.x/zh/guide/subnet/index.html
+++ b/v1.13.x/zh/guide/subnet/index.html
@@ -147,13 +147,13 @@
 <span class=w>  </span><span class=nt>name</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">disable-gw-check</span>
 <span class=nt>spec</span><span class=p>:</span>
 <span class=w>  </span><span class=nt>disableGatewayCheck</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
-</code></pre></div> <h2 id=multicast-snoop>Multicast-Snoop 配置<a class=headerlink href=#multicast-snoop title="Permanent link">&para;</a></h2> <p>默认情况下 subnet 下的 Pod 如果发送组播报文，OVN 的默认行为是会广播组播报文到子网下所有的 Pod。如果开启subnet 的 multicast snoop 开关，OVN 会根据 <code>South Database</code> 中的组播表 <code>Multicast_Group</code> 来进行转发，而不在进行广播。</p> <div class=highlight><pre><span></span><code><span class=nt>apiVersion</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">kubeovn.io/v1</span>
+</code></pre></div> <h2 id=multicast-snoop>Multicast-Snoop 配置<a class=headerlink href=#multicast-snoop title="Permanent link">&para;</a></h2> <p>默认情况下 subnet 下的 Pod 如果发送组播报文，OVN 的默认行为是会广播组播报文到子网下所有的 Pod。如果开启 subnet 的 multicast snoop 开关，OVN 会根据 <code>South Database</code> 中的组播表 <code>Multicast_Group</code> 来进行转发，而不在进行广播。</p> <div class=highlight><pre><span></span><code><span class=nt>apiVersion</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">kubeovn.io/v1</span>
 <span class=nt>kind</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">Subnet</span>
 <span class=nt>metadata</span><span class=p>:</span>
 <span class=w>  </span><span class=nt>name</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">sample1</span>
 <span class=nt>spec</span><span class=p>:</span>
 <span class=w>  </span><span class=nt>enableMulticastSnoop</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
-</code></pre></div> <h2 id=_12>其他高级设置<a class=headerlink href=#_12 title="Permanent link">&para;</a></h2> <ul> <li><a href=../ippool/ >IP 池使用</a></li> <li><a href=../../advance/nat-policy-rule/ >默认 VPC NAT 策略规则</a></li> <li><a href=../qos/ >QoS 设置</a></li> <li><a href=../../advance/multi-nic/ >多网卡管理</a></li> <li><a href=../../advance/dhcp/ >DHCP 选项</a></li> <li><a href=../../advance/external-gateway/ >外部网关设置</a></li> <li><a href=../../advance/with-ovn-ic/ >集群互联设置</a></li> <li><a href=../../advance/vip/ >虚拟 IP 设置</a></li> </ul> <p><a class=md-button href=https://ma.alauda.cn/p/0f39a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M9.5 4C5.36 4 2 6.69 2 10c0 1.89 1.08 3.56 2.78 4.66L4 17l2.5-1.5c.89.31 1.87.5 2.91.5A5.22 5.22 0 0 1 9 14c0-3.31 3.13-6 7-6 .19 0 .38 0 .56.03C15.54 5.69 12.78 4 9.5 4m-3 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m5 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1M16 9c-3.31 0-6 2.24-6 5s2.69 5 6 5c.67 0 1.31-.08 1.91-.25L20 20l-.62-1.87C20.95 17.22 22 15.71 22 14c0-2.76-2.69-5-6-5m-2 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m4 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1Z"/></svg></span> 微信群</a> <a class=md-button href=https://communityinviter.com/apps/kube-ovn/kube-ovn/ target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M6 15a2 2 0 0 1-2 2 2 2 0 0 1-2-2 2 2 0 0 1 2-2h2v2m1 0a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-5m2-8a2 2 0 0 1-2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2v2H9m0 1a2 2 0 0 1 2 2 2 2 0 0 1-2 2H4a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5m8 2a2 2 0 0 1 2-2 2 2 0 0 1 2 2 2 2 0 0 1-2 2h-2v-2m-1 0a2 2 0 0 1-2 2 2 2 0 0 1-2-2V5a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5m-2 8a2 2 0 0 1 2 2 2 2 0 0 1-2 2 2 2 0 0 1-2-2v-2h2m0-1a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-5Z"/></svg></span> Slack</a> <a class=md-button href=https://twitter.com/KubeOvn target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M22.46 6c-.77.35-1.6.58-2.46.69.88-.53 1.56-1.37 1.88-2.38-.83.5-1.75.85-2.72 1.05C18.37 4.5 17.26 4 16 4c-2.35 0-4.27 1.92-4.27 4.29 0 .34.04.67.11.98C8.28 9.09 5.11 7.38 3 4.79c-.37.63-.58 1.37-.58 2.15 0 1.49.75 2.81 1.91 3.56-.71 0-1.37-.2-1.95-.5v.03c0 2.08 1.48 3.82 3.44 4.21a4.22 4.22 0 0 1-1.93.07 4.28 4.28 0 0 0 4 2.98 8.521 8.521 0 0 1-5.33 1.84c-.34 0-.68-.02-1.02-.06C3.44 20.29 5.7 21 8.12 21 16 21 20.33 14.46 20.33 8.79c0-.19 0-.37-.01-.56.84-.6 1.56-1.36 2.14-2.23Z"/></svg></span> Twitter</a> <a class=md-button href=https://ma.alauda.cn/p/2f53a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="m20 8-8 5-8-5V6l8 5 8-5m0-2H4c-1.11 0-2 .89-2 2v12a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V6a2 2 0 0 0-2-2Z"/></svg></span> Support</a></p> <hr> <div class=md-source-file> <small> 最后更新: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2023年9月19日</span> <br> 创建日期: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2022年5月24日</span> </small> </div> <h2 id=__comments>评论</h2> <script src=https://giscus.app/client.js data-repo=kubeovn/kube-ovn data-repo-id="MDEwOlJlcG9zaXRvcnkxNzcwNjg5NjE=" data-category=Announcements data-category-id=DIC_kwDOCo3boc4CAj60 data-mapping=pathname data-reactions-enabled=1 data-emit-metadata=0 data-input-position=bottom data-theme=light data-lang=zh-CN crossorigin=anonymous async>
+</code></pre></div> <h2 id=_12>其他高级设置<a class=headerlink href=#_12 title="Permanent link">&para;</a></h2> <ul> <li><a href=../ippool/ >IP 池使用</a></li> <li><a href=../../advance/nat-policy-rule/ >默认 VPC NAT 策略规则</a></li> <li><a href=../qos/ >QoS 设置</a></li> <li><a href=../../advance/multi-nic/ >多网卡管理</a></li> <li><a href=../../advance/dhcp/ >DHCP 选项</a></li> <li><a href=../../advance/external-gateway/ >外部网关设置</a></li> <li><a href=../../advance/with-ovn-ic/ >集群互联设置</a></li> <li><a href=../../advance/vip/ >虚拟 IP 设置</a></li> </ul> <p><a class=md-button href=https://ma.alauda.cn/p/0f39a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M9.5 4C5.36 4 2 6.69 2 10c0 1.89 1.08 3.56 2.78 4.66L4 17l2.5-1.5c.89.31 1.87.5 2.91.5A5.22 5.22 0 0 1 9 14c0-3.31 3.13-6 7-6 .19 0 .38 0 .56.03C15.54 5.69 12.78 4 9.5 4m-3 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m5 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1M16 9c-3.31 0-6 2.24-6 5s2.69 5 6 5c.67 0 1.31-.08 1.91-.25L20 20l-.62-1.87C20.95 17.22 22 15.71 22 14c0-2.76-2.69-5-6-5m-2 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m4 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1Z"/></svg></span> 微信群</a> <a class=md-button href=https://communityinviter.com/apps/kube-ovn/kube-ovn/ target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M6 15a2 2 0 0 1-2 2 2 2 0 0 1-2-2 2 2 0 0 1 2-2h2v2m1 0a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-5m2-8a2 2 0 0 1-2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2v2H9m0 1a2 2 0 0 1 2 2 2 2 0 0 1-2 2H4a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5m8 2a2 2 0 0 1 2-2 2 2 0 0 1 2 2 2 2 0 0 1-2 2h-2v-2m-1 0a2 2 0 0 1-2 2 2 2 0 0 1-2-2V5a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5m-2 8a2 2 0 0 1 2 2 2 2 0 0 1-2 2 2 2 0 0 1-2-2v-2h2m0-1a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-5Z"/></svg></span> Slack</a> <a class=md-button href=https://twitter.com/KubeOvn target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M22.46 6c-.77.35-1.6.58-2.46.69.88-.53 1.56-1.37 1.88-2.38-.83.5-1.75.85-2.72 1.05C18.37 4.5 17.26 4 16 4c-2.35 0-4.27 1.92-4.27 4.29 0 .34.04.67.11.98C8.28 9.09 5.11 7.38 3 4.79c-.37.63-.58 1.37-.58 2.15 0 1.49.75 2.81 1.91 3.56-.71 0-1.37-.2-1.95-.5v.03c0 2.08 1.48 3.82 3.44 4.21a4.22 4.22 0 0 1-1.93.07 4.28 4.28 0 0 0 4 2.98 8.521 8.521 0 0 1-5.33 1.84c-.34 0-.68-.02-1.02-.06C3.44 20.29 5.7 21 8.12 21 16 21 20.33 14.46 20.33 8.79c0-.19 0-.37-.01-.56.84-.6 1.56-1.36 2.14-2.23Z"/></svg></span> Twitter</a> <a class=md-button href=https://ma.alauda.cn/p/2f53a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="m20 8-8 5-8-5V6l8 5 8-5m0-2H4c-1.11 0-2 .89-2 2v12a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V6a2 2 0 0 0-2-2Z"/></svg></span> Support</a></p> <hr> <div class=md-source-file> <small> 最后更新: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2023年9月26日</span> <br> 创建日期: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2022年5月24日</span> </small> </div> <h2 id=__comments>评论</h2> <script src=https://giscus.app/client.js data-repo=kubeovn/kube-ovn data-repo-id="MDEwOlJlcG9zaXRvcnkxNzcwNjg5NjE=" data-category=Announcements data-category-id=DIC_kwDOCo3boc4CAj60 data-mapping=pathname data-reactions-enabled=1 data-emit-metadata=0 data-input-position=bottom data-theme=light data-lang=zh-CN crossorigin=anonymous async>
 </script> <script>
     var palette = __md_get("__palette")
     if (palette && typeof palette.color === "object")
diff --git a/v1.13.x/zh/guide/vpc/index.html b/v1.13.x/zh/guide/vpc/index.html
index 6eedbf81c..6a66896fa 100644
--- a/v1.13.x/zh/guide/vpc/index.html
+++ b/v1.13.x/zh/guide/vpc/index.html
@@ -182,7 +182,7 @@
 <span class=nt>spec</span><span class=p>:</span>
 <span class=w>  </span><span class=nt>eip</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">eips01</span>
 <span class=w>  </span><span class=nt>internalCIDR</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">10.0.1.0/24</span>
-</code></pre></div> <h3 id=ip>创建浮动 IP<a class=headerlink href=#ip title="Permanent link">&para;</a></h3> <p>通过浮动 IP 规则，VPC 内的一个 IP 会和 EIP 进行完全映射，外部可以通过这个 EIP 方位 VPC 内的 IP，VPC 内的这个 IP 访问外部地址时也会 SNAT 成这个 EIP。</p> <div class=highlight><pre><span></span><code><span class=nn>---</span>
+</code></pre></div> <h3 id=ip>创建浮动 IP<a class=headerlink href=#ip title="Permanent link">&para;</a></h3> <p>通过浮动 IP 规则，VPC 内的一个 IP 会和 EIP 进行完全映射，外部可以通过这个 EIP 访问 VPC 内的 IP，VPC 内的这个 IP 访问外部地址时也会 SNAT 成这个 EIP。</p> <div class=highlight><pre><span></span><code><span class=nn>---</span>
 <span class=nt>kind</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">IptablesEIP</span>
 <span class=nt>apiVersion</span><span class=p>:</span><span class=w> </span><span class="l l-Scalar l-Scalar-Plain">kubeovn.io/v1</span>
 <span class=nt>metadata</span><span class=p>:</span>
@@ -370,7 +370,7 @@
 NAME<span class=w>        </span>ACTIVE<span class=w>   </span>VPC<span class=w>         </span>SUBNET<span class=w>   </span>
 test-cjh1<span class=w>   </span><span class=nb>false</span><span class=w>    </span>cjh-vpc-1<span class=w>   </span>cjh-subnet-1<span class=w>   </span>
 test-cjh2<span class=w>   </span><span class=nb>true</span><span class=w>     </span>cjh-vpc-1<span class=w>   </span>cjh-subnet-2<span class=w> </span>
-</code></pre></div> <ul> <li><code>ACTIVE</code>: <code>true</code> 成功部署了自定义 dns 组件，<code>false</code> 无部署</li> </ul> <h3 id=_7>限制<a class=headerlink href=#_7 title="Permanent link">&para;</a></h3> <ul> <li>一个 vpc 下只会部署一个自定义 dns 组件;</li> <li>当一个 vpc 下配置多个 vpc-dns 资源（即同一个 vpc 不同的 subnet），只有一个 vpc-dns 资源状态 <code>true</code>，其他为 <code>fasle</code>;</li> <li>当 <code>true</code> 的 vpc-dns 被删除掉，会获取其他 <code>false</code> 的 vpc-dns 进行部署。</li> </ul> <p><a class=md-button href=https://ma.alauda.cn/p/0f39a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M9.5 4C5.36 4 2 6.69 2 10c0 1.89 1.08 3.56 2.78 4.66L4 17l2.5-1.5c.89.31 1.87.5 2.91.5A5.22 5.22 0 0 1 9 14c0-3.31 3.13-6 7-6 .19 0 .38 0 .56.03C15.54 5.69 12.78 4 9.5 4m-3 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m5 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1M16 9c-3.31 0-6 2.24-6 5s2.69 5 6 5c.67 0 1.31-.08 1.91-.25L20 20l-.62-1.87C20.95 17.22 22 15.71 22 14c0-2.76-2.69-5-6-5m-2 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m4 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1Z"/></svg></span> 微信群</a> <a class=md-button href=https://communityinviter.com/apps/kube-ovn/kube-ovn/ target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M6 15a2 2 0 0 1-2 2 2 2 0 0 1-2-2 2 2 0 0 1 2-2h2v2m1 0a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-5m2-8a2 2 0 0 1-2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2v2H9m0 1a2 2 0 0 1 2 2 2 2 0 0 1-2 2H4a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5m8 2a2 2 0 0 1 2-2 2 2 0 0 1 2 2 2 2 0 0 1-2 2h-2v-2m-1 0a2 2 0 0 1-2 2 2 2 0 0 1-2-2V5a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5m-2 8a2 2 0 0 1 2 2 2 2 0 0 1-2 2 2 2 0 0 1-2-2v-2h2m0-1a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-5Z"/></svg></span> Slack</a> <a class=md-button href=https://twitter.com/KubeOvn target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M22.46 6c-.77.35-1.6.58-2.46.69.88-.53 1.56-1.37 1.88-2.38-.83.5-1.75.85-2.72 1.05C18.37 4.5 17.26 4 16 4c-2.35 0-4.27 1.92-4.27 4.29 0 .34.04.67.11.98C8.28 9.09 5.11 7.38 3 4.79c-.37.63-.58 1.37-.58 2.15 0 1.49.75 2.81 1.91 3.56-.71 0-1.37-.2-1.95-.5v.03c0 2.08 1.48 3.82 3.44 4.21a4.22 4.22 0 0 1-1.93.07 4.28 4.28 0 0 0 4 2.98 8.521 8.521 0 0 1-5.33 1.84c-.34 0-.68-.02-1.02-.06C3.44 20.29 5.7 21 8.12 21 16 21 20.33 14.46 20.33 8.79c0-.19 0-.37-.01-.56.84-.6 1.56-1.36 2.14-2.23Z"/></svg></span> Twitter</a> <a class=md-button href=https://ma.alauda.cn/p/2f53a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="m20 8-8 5-8-5V6l8 5 8-5m0-2H4c-1.11 0-2 .89-2 2v12a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V6a2 2 0 0 0-2-2Z"/></svg></span> Support</a></p> <hr> <div class=md-source-file> <small> 最后更新: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2023年8月17日</span> <br> 创建日期: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2022年5月24日</span> </small> </div> <h2 id=__comments>评论</h2> <script src=https://giscus.app/client.js data-repo=kubeovn/kube-ovn data-repo-id="MDEwOlJlcG9zaXRvcnkxNzcwNjg5NjE=" data-category=Announcements data-category-id=DIC_kwDOCo3boc4CAj60 data-mapping=pathname data-reactions-enabled=1 data-emit-metadata=0 data-input-position=bottom data-theme=light data-lang=zh-CN crossorigin=anonymous async>
+</code></pre></div> <ul> <li><code>ACTIVE</code>: <code>true</code> 成功部署了自定义 dns 组件，<code>false</code> 无部署</li> </ul> <h3 id=_7>限制<a class=headerlink href=#_7 title="Permanent link">&para;</a></h3> <ul> <li>一个 vpc 下只会部署一个自定义 dns 组件;</li> <li>当一个 vpc 下配置多个 vpc-dns 资源（即同一个 vpc 不同的 subnet），只有一个 vpc-dns 资源状态 <code>true</code>，其他为 <code>fasle</code>;</li> <li>当 <code>true</code> 的 vpc-dns 被删除掉，会获取其他 <code>false</code> 的 vpc-dns 进行部署。</li> </ul> <p><a class=md-button href=https://ma.alauda.cn/p/0f39a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M9.5 4C5.36 4 2 6.69 2 10c0 1.89 1.08 3.56 2.78 4.66L4 17l2.5-1.5c.89.31 1.87.5 2.91.5A5.22 5.22 0 0 1 9 14c0-3.31 3.13-6 7-6 .19 0 .38 0 .56.03C15.54 5.69 12.78 4 9.5 4m-3 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m5 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1M16 9c-3.31 0-6 2.24-6 5s2.69 5 6 5c.67 0 1.31-.08 1.91-.25L20 20l-.62-1.87C20.95 17.22 22 15.71 22 14c0-2.76-2.69-5-6-5m-2 2.5a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1m4 0a1 1 0 0 1 1 1 1 1 0 0 1-1 1 1 1 0 0 1-1-1 1 1 0 0 1 1-1Z"/></svg></span> 微信群</a> <a class=md-button href=https://communityinviter.com/apps/kube-ovn/kube-ovn/ target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M6 15a2 2 0 0 1-2 2 2 2 0 0 1-2-2 2 2 0 0 1 2-2h2v2m1 0a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-5m2-8a2 2 0 0 1-2-2 2 2 0 0 1 2-2 2 2 0 0 1 2 2v2H9m0 1a2 2 0 0 1 2 2 2 2 0 0 1-2 2H4a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5m8 2a2 2 0 0 1 2-2 2 2 0 0 1 2 2 2 2 0 0 1-2 2h-2v-2m-1 0a2 2 0 0 1-2 2 2 2 0 0 1-2-2V5a2 2 0 0 1 2-2 2 2 0 0 1 2 2v5m-2 8a2 2 0 0 1 2 2 2 2 0 0 1-2 2 2 2 0 0 1-2-2v-2h2m0-1a2 2 0 0 1-2-2 2 2 0 0 1 2-2h5a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-5Z"/></svg></span> Slack</a> <a class=md-button href=https://twitter.com/KubeOvn target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M22.46 6c-.77.35-1.6.58-2.46.69.88-.53 1.56-1.37 1.88-2.38-.83.5-1.75.85-2.72 1.05C18.37 4.5 17.26 4 16 4c-2.35 0-4.27 1.92-4.27 4.29 0 .34.04.67.11.98C8.28 9.09 5.11 7.38 3 4.79c-.37.63-.58 1.37-.58 2.15 0 1.49.75 2.81 1.91 3.56-.71 0-1.37-.2-1.95-.5v.03c0 2.08 1.48 3.82 3.44 4.21a4.22 4.22 0 0 1-1.93.07 4.28 4.28 0 0 0 4 2.98 8.521 8.521 0 0 1-5.33 1.84c-.34 0-.68-.02-1.02-.06C3.44 20.29 5.7 21 8.12 21 16 21 20.33 14.46 20.33 8.79c0-.19 0-.37-.01-.56.84-.6 1.56-1.36 2.14-2.23Z"/></svg></span> Twitter</a> <a class=md-button href=https://ma.alauda.cn/p/2f53a target=_blank><span class=twemoji><svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="m20 8-8 5-8-5V6l8 5 8-5m0-2H4c-1.11 0-2 .89-2 2v12a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V6a2 2 0 0 0-2-2Z"/></svg></span> Support</a></p> <hr> <div class=md-source-file> <small> 最后更新: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2023年9月26日</span> <br> 创建日期: <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">2022年5月24日</span> </small> </div> <h2 id=__comments>评论</h2> <script src=https://giscus.app/client.js data-repo=kubeovn/kube-ovn data-repo-id="MDEwOlJlcG9zaXRvcnkxNzcwNjg5NjE=" data-category=Announcements data-category-id=DIC_kwDOCo3boc4CAj60 data-mapping=pathname data-reactions-enabled=1 data-emit-metadata=0 data-input-position=bottom data-theme=light data-lang=zh-CN crossorigin=anonymous async>
 </script> <script>
     var palette = __md_get("__palette")
     if (palette && typeof palette.color === "object")