package org.kne.cloud.network.klalb; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintStream; import java.net.Inet6Address; import java.net.InetAddress; import java.net.SocketTimeoutException; import java.nio.channels.UnresolvedAddressException; import java.security.SecureRandom; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.PriorityBlockingQueue; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.LockSupport; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; import java.util.function.Supplier; import org.kne.cloud.clock.AdjustedNanoClock; import org.kne.cloud.clock.ReliabilityBackoffTimeClock; import org.kne.cloud.network.MultipurposeSocketAddress; import org.kne.cloud.network.SpeedLimiter; import org.kne.cloud.network.ThreadTool; import org.kne.cloud.network.ipv6.IPv6NetworkLink; import org.kne.cloud.network.ipv6.IPv6Packet; import org.kne.cloud.network.ipv6.Inet6AddressGroup; import org.kne.cloud.network.ipv6.Neighbor; import org.kne.cloud.network.monitor.MonitorData; import org.kne.cloud.network.monitor.QueueingMonitorDataImpl; import org.kne.cloud.network.monitor.SpeedAndTrafficAndDelayMonitorDataImpl; import org.kne.cloud.network.te.LoadingBalanceEntry; public class KLALBRemoteLine implements IPv6NetworkLink,Comparable ,LoadingBalanceEntry{ private static final boolean debug = false; private static final boolean showpacket = false; private ReliabilityBackoffTimeClock coll = new ReliabilityBackoffTimeClock(); private volatile boolean closed = false; private volatile Supplier localVaddrSupplier; private volatile KLALBController klalbController; public KLALBController getKlalbController() { return klalbController; } public void setKlalbController(KLALBController klalbController) { this.klalbController = klalbController; if (klalbController != null && remoteVaddr != null) { adjnc = klalbController.getAdjustedClockByVaddr(remoteVaddr.getAddress()); } } public Supplier getLocalVaddrSupplier() { return localVaddrSupplier; } public void setLocalVaddrSupplier(Supplier localVaddrSupplier) { this.localVaddrSupplier = localVaddrSupplier; } private volatile Inet6AddressGroup remoteVaddr; public Inet6AddressGroup getRemoteVaddr() { return remoteVaddr; } private volatile AdjustedNanoClock adjnc = new AdjustedNanoClock(); public void waitForRemoteVaddrAvaliable(long timeout) throws SocketTimeoutException { long s = System.nanoTime(); while (remoteVaddr == null) { try { Thread.sleep(1); } catch (InterruptedException e) { e.printStackTrace(); } if (System.nanoTime() - s > timeout * 1000000) throw new SocketTimeoutException(); } } public QueueingMonitorDataImpl getMonitor() { return monitor; } public String toString() { return remoteVaddr + "\t" + monitor.toString(); } private QueueingMonitorDataImpl monitor; private volatile KLALBPacketLink kplink; private MultipurposeSocketAddress bindAddress; private MultipurposeSocketAddress socketAddress; private Thread lthd; public void setKplink(KLALBPacketLink kplink) { this.kplink = kplink; } public KLALBPacketLink getKplink() { return kplink; } public MultipurposeSocketAddress getSocketAddress() { return socketAddress; } public MultipurposeSocketAddress getBindAddress() { return bindAddress; } private Inet6AddressGroup addressGroup; private Inet6AddressGroup peerAddress; @Override public Inet6AddressGroup getAddressGroup() { return addressGroup; } public Inet6AddressGroup getPeerAddress() { return peerAddress; } public KLALBRemoteLine(MultipurposeSocketAddress mpa, Inet6AddressGroup address) { this(mpa,address, new QueueingMonitorDataImpl()); } public KLALBRemoteLine(MultipurposeSocketAddress mpa, Inet6AddressGroup address, QueueingMonitorDataImpl monitor) { this(mpa, null,address, monitor); } public KLALBRemoteLine(KLALBPacketLink kpl, Inet6AddressGroup address) { this(kpl,address, new QueueingMonitorDataImpl()); } public KLALBRemoteLine(KLALBPacketLink kpl, Inet6AddressGroup address,QueueingMonitorDataImpl monitor) { this.kplink = kpl; this.addressGroup=address; this.monitor = monitor; monitor.setName(kpl.toString()); } public KLALBRemoteLine(MultipurposeSocketAddress mpa, MultipurposeSocketAddress bindaddr,Inet6AddressGroup address, QueueingMonitorDataImpl monitor) { this.socketAddress = mpa; this.bindAddress = bindaddr; this.addressGroup=address; this.monitor = monitor; if (bindaddr == null) { monitor.setName("→" + mpa.toString()); } else { monitor.setName(bindaddr.toString() + "→" + mpa.toString()); } } public KLALBRemoteLine(MultipurposeSocketAddress mpa, MultipurposeSocketAddress bindaddr, Inet6AddressGroup address) { this(mpa, bindaddr,address, new QueueingMonitorDataImpl()); } public KLALBRemoteLine(MultipurposeSocketAddress mpa, MultipurposeSocketAddress bind) { this(mpa, bind, null); } public KLALBRemoteLine(MultipurposeSocketAddress mpa) { this(mpa,(Inet6AddressGroup)null); } public KLALBRemoteLine(KLALBPacketLink link) { this(link, (Inet6AddressGroup)null); } PrintStream pw; private boolean flag=false; private void startRecord() { if(flag) return; flag=true; try { File fl = new File("logs"); if (!fl.exists()) { fl.mkdirs(); } File fx = new File(fl, getMonitor().getName().replace(':', ':') + ".csv"); pw = new PrintStream(new FileOutputStream(fx, true)); if (fx.length() <= 0) pw.println("时间,状态,上传速度,下载速度,上传延迟,下载延迟,往返延迟,抖动"); ThreadTool.makeVDaemonThreadIfSupport("BigData Record", () -> { while (!closed) { try { Thread.sleep(100); } catch (InterruptedException e) { e.printStackTrace(); } pw.println(System.currentTimeMillis() + "," + monitor.getState() + "," + monitor.getOutSpeed() + "," + monitor.getInSpeed() + "," + monitor.getOutDelay() + "," + monitor.getInDelay() + "," + monitor.getLatency() + "," + monitor.getTotalJitter()); } pw.close(); }).start(); } catch (FileNotFoundException e) { e.printStackTrace(); } } private static long MIN_SPEED=256*1024L; private SpeedLimiter congress = new SpeedLimiter(MIN_SPEED,10000000L); private long congressSpeed = 0; private double loadPercent = 1.01; protected void startIO() { ThreadTool.makeVDaemonThreadIfSupport("远程接收线程", () -> { //Thread.currentThread().setPriority(Thread.NORM_PRIORITY+1); while (true) { try { monitor.setState(MonitorData.CONNECTING); if (socketAddress == null) { if (kplink.isClosed()) { close(); break; } if(addressGroup==null) { byte[]addr=new byte[16]; SecureRandom scr=new SecureRandom(); scr.nextBytes(addr); addr[0]=(byte)0x24; addr[1]=(byte) 0x86; addr[2]=0; addr[3]=2; addr[14]=0; addr[15]=1; Inet6Address i6a=(Inet6Address) Inet6Address.getByAddress(addr); addressGroup=new Inet6AddressGroup(i6a, 112); } } else { kplink = KLALBUtils. createKLALBPacketLink(bindAddress, socketAddress); } //startRecord(); kplink.setSoTimeout(20000); Thread t = ThreadTool.makeVDaemonThreadIfSupport("远程发送线程", () -> { tlock = Thread.currentThread(); //tlock.setPriority(Thread.NORM_PRIORITY+1); try { writePacketToKPL(new ADDRREQPacket()); writePacketToKPL(new ADDRREQPacket()); if(addressGroup!=null) { writePacketToKPL(new ADDRPacket(addressGroup)); writePacketToKPL(new ADDRPacket(addressGroup)); } writePacketToKPL(new VADDRREQPacket()); writePacketToKPL(new VADDRREQPacket()); writePacketToKPL(new VADDRPacket(localVaddrSupplier.get())); writePacketToKPL(new VADDRPacket(localVaddrSupplier.get())); flushKPL(); KLALBPacket kpp = null; while ((!kplink.isClosed()) && (!closed)) { // TimeDebugger tdb=new TimeDebugger(); // tdb.putTime("start"); if(checkBandwidthReportTime()) { sendIPacket(new BWINFPacket(monitor.getOutSpeedAvg2(), monitor.getInSpeedAvg2())); } if (checkPingTimeSleep()) { writePacketToKPL(new PINGPacket(System.nanoTime())); if (remoteVaddr == null) writePacketToKPL(new VADDRREQPacket()); if(peerAddress==null) writePacketToKPL(new ADDRREQPacket()); }else { KLALBPacket kpip = IsendDequeList.poll(); if (kpip != null) { writePacketToKPL(kpip); }else { // tdb.putTime("Isend"); if (kpp == null) { do { kpp = sendDequeList.poll(); } while (kpp != null && kpp.isDisposed()); if(kpp==null) { monitor.setQueueingDelay(0); } } // System.out.println(sendDequeList.size()); if (kpp != null) { // TimeDebugger tdb=new TimeDebugger(); //tdb.putTime("start"); KLALBPacket kppt=kpp; kppt.lockAll(); try { if (kpp.isSomeDisposed()) { kpp = null; } else { // if (congress.checkTransmit(length)) { writePacketToKPL(kpp); if (checkPingTime()) { writePacketToKPL(new PINGPacket(System.nanoTime())); } monitor.setQueueingDelay( System.nanoTime()- kpp.getJoinqueuetime()); /*if (kpp instanceof DATATPacket||kpp instanceof IPv6OverKLALBPacket) { resetSleepTimer(); }*/ kpp.doDisposeAfterSend(); kpp = null; } //} } finally { kppt.unlockAll(); } //tdb.putTime("sendToLink"); //tdb.print(); }else { LockSupport.parkNanos(1000000); } } } /*if (flsh) { flushKPL(); } else { // tdb.putTime("send"); // tdb.putTime("flush"); if (pressure && pressureSpeed.checkTransmit(testPacket.getLength())) { writePacketToKPL(testPacket); flushKPL(); } else { tlock = Thread.currentThread(); LockSupport.parkNanos(1000000); } // tdb.putTime("park"); }*/ // tdb.print(); // System.out.println(sendDequeList.isEmpty()); Thread.yield(); } } catch (IOException | UnresolvedAddressException e) { if (debug) e.printStackTrace(); } finally { try { kplink.close(); } catch (IOException e) { e.printStackTrace(); } } }); t.start(); boolean fst = true; long stime = 5000000000L; while ((!kplink.isClosed()) && (!closed)) { long readStart = System.nanoTime(); KLALBPacket kpp = readPacketFromKPL(); /*long readTime = System.nanoTime() - readStart; readTime *= 10; if (readTime > stime) { stime = readTime; } else { stime = (stime * 999 + readTime) / 1000; } if (stime < 2000000000L) { stime = 2000000000L; }*/ // System.out.println(readTime); kplink.setSoTimeout(3000); kplink.setSoTimeout((int) (stime / 1000000)); if (kpp == null) { break; } switch (kpp.getType()) { case KLALBPacket.PING: sendIPacket(new PONGPacket(((PINGPacket) kpp).getTime() ,kpp.getRcvtime(), System.nanoTime())); break; case KLALBPacket.PONG: PONGPacket png = (PONGPacket) kpp; long ul = png.getRcvtime() - png.getTimepingsnd()-(png.getTimepongsnd()-png.getTimepingrcv()); long DsndDelayFactor0 = png.getTimepingrcv() - png.getTimepingsnd(); long DrcvDelayFactor0 = png.getRcvtime() - png.getTimepongsnd(); long DsndDelayFactor; long DrcvDelayFactor; adjnc.getLock().lock(); try { adjnc.calibrate(png.getTimepongsnd() + ul / 2 - png.getRcvtime(), ul / 2); DsndDelayFactor = DsndDelayFactor0 - adjnc.getDelta(); DrcvDelayFactor = DrcvDelayFactor0 + adjnc.getDelta(); if (DsndDelayFactor < 0) { if (DrcvDelayFactor < 0) { } else { adjnc.setDelta(adjnc.getDelta() + DsndDelayFactor); DsndDelayFactor = DsndDelayFactor0 - adjnc.getDelta(); DrcvDelayFactor = DrcvDelayFactor0 + adjnc.getDelta(); } } else { if (DrcvDelayFactor < 0) { adjnc.setDelta(adjnc.getDelta() - DrcvDelayFactor); DsndDelayFactor = DsndDelayFactor0 - adjnc.getDelta(); DrcvDelayFactor = DrcvDelayFactor0 + adjnc.getDelta(); } } } finally { adjnc.getLock().unlock(); } if (DsndDelayFactor >= 0 && DrcvDelayFactor >= 0) { monitor.setOutDelay(DsndDelayFactor); monitor.setInDelay(DrcvDelayFactor); monitor.setRecentPingNanoTime(png.getTimepingsnd()); //pingInterval = monitor.getLatencyAvg()+2000000L; //double load = 1 - monitor.getOutDelayMin() / (double) monitor.getOutDelay(); loadPercent =Math.max( 1.2,0.4); } LockSupport.unpark(tlock); break; case KLALBPacket.BWINF: BWINFPacket bwi = (BWINFPacket) kpp; if (bwi.getDownSpeed() >= congressSpeed) { congressSpeed = bwi.getDownSpeed(); } else { congressSpeed = (congressSpeed * 999 + bwi.getDownSpeed()) / 1000; } congress.setLimitspeed(Math.max(MIN_SPEED,(long) (congressSpeed * loadPercent)) ); // System.out.println(congressSpeed); break; case KLALBPacket.ADDRREQ: if(addressGroup!=null) sendIPacket(new ADDRPacket(addressGroup)); break; case KLALBPacket.ADDR: peerAddress = ((ADDRPacket) kpp).getAddr(); if(addressGroup==null) { byte[]ab=peerAddress.getAddress().getAddress(); ab[15]=2; Inet6AddressGroup ardg2=new Inet6AddressGroup((Inet6Address) InetAddress.getByAddress(ab),peerAddress.getPrefixLength()); addressGroup=ardg2; } break; case KLALBPacket.VADDRREQ: sendIPacket(new VADDRPacket(localVaddrSupplier.get())); break; case KLALBPacket.VADDR: Inet6AddressGroup vdr = ((VADDRPacket) kpp).getVaddr(); remoteVaddr = vdr; sendIPacket(new VADDRACKPacket()); if (klalbController != null) { adjnc = klalbController.getAdjustedClockByVaddr(remoteVaddr.getAddress()); } if (fst) { // coll.resetCoolingTime(); monitor.setState(MonitorData.ONLINE); fst = false; } break; case KLALBPacket.VADDRACK: break; case KLALBPacket.TEST: break; case KLALBPacket.IPV6OVERKLALB: if(ipv6con!=null) { IPv6OverKLALBPacket ivk=(IPv6OverKLALBPacket) kpp; IPv6Packet iv6= ivk.getIPv6Packet(); ivk.putTimePassport("unpacked"); ivk.printPassport(); iv6.putTimePassport("unpackFromLink"); ipv6con.accept(iv6); } break; default: while (rec == null) { Thread.sleep(1); } rec.accept( kpp); break; } Thread.yield(); } } catch (IOException | UnresolvedAddressException e) { if (debug) e.printStackTrace(); } catch (InterruptedException e) { e.printStackTrace(); } finally { monitor.setState(MonitorData.OFFLINE); pressure = false; try { if (kplink != null) kplink.close(); } catch (IOException e) { e.printStackTrace(); } /*KLALBPacket pack; while((pack=sendDequeList.poll())!=null) { try { if(remoteVaddr!=null) { klalbController.sendPacketToLinkAddress(remoteVaddr, pack); if(debug) System.out.println("RETRY:"+pack); } } catch (IOException e) { } }*/ } if (closed) { break; } // Thread.sleep(monitor.getCoolingTime()); lthd = Thread.currentThread(); LockSupport.parkNanos(coll.getCoolingTime(monitor.getReliability())); // coll.incCoolingTime(); } }).start(); } private void flushKPL() throws IOException { kplink.flush(); } private KLALBPacket readPacketFromKPL() throws IOException { KLALBPacket packet = kplink.readPacket(); if (showpacket) { if (!(packet instanceof PINGPacket)) if (!(packet instanceof PONGPacket)) if (!(packet instanceof BWINFPacket)) System.out.println("RX:" + packet); } if (packet != null) { packet.putTimePassport("received"); long length=packet.getLength(); monitor.getInTrafficAL().addAndGet(length); monitor.getInPacketCounterAL().incrementAndGet(); if (klalbController != null) klalbController.getLinkMonitor().getInTrafficAL().addAndGet(length); klalbController.getLinkMonitor().getInPacketCounterAL().incrementAndGet(); } return packet; } private void writePacketToKPL(KLALBPacket packet) throws IOException { long length=packet.getLength(); monitor.getOutTrafficAL().addAndGet(length); monitor.getOutPacketCounterAL().incrementAndGet(); if (klalbController != null) klalbController.getLinkMonitor().getOutTrafficAL().addAndGet(length); klalbController.getLinkMonitor().getOutPacketCounterAL().incrementAndGet(); kplink.writePacket(packet); packet.putTimePassport("sended"); packet.printPassport(); if (showpacket) { if (!(packet instanceof PINGPacket)) if (!(packet instanceof PONGPacket)) if (!(packet instanceof BWINFPacket)) System.err.println("TX:" + packet); } } private void writePacketToKPL(KLALBPacket packet,long length) throws IOException { monitor.getOutTrafficAL().addAndGet(length); monitor.getOutPacketCounterAL().incrementAndGet(); if (klalbController != null) klalbController.getLinkMonitor().getOutTrafficAL().addAndGet(length); klalbController.getLinkMonitor().getOutPacketCounterAL().incrementAndGet(); kplink.writePacket(packet); if (showpacket) { if (!(packet instanceof PINGPacket)) if (!(packet instanceof PONGPacket)) if (!(packet instanceof BWINFPacket)) System.err.println("TX:" + packet); } } public void close() { closed = true; monitor.setState(MonitorData.OFFLINE); pressure = false; try { if (kplink != null) kplink.close(); } catch (IOException e) { e.printStackTrace(); } } public boolean isClosed() { return closed; } private volatile long time = System.nanoTime(); private volatile long pingInterval = 10000000L; private volatile long pingIntervalSleep = 200000000L; private boolean checkPingTime() { long cu = System.nanoTime(); if (cu - time > pingInterval) { time = cu; return true; } else { return false; } } private boolean checkPingTimeSleep() { long cu = System.nanoTime(); if (cu - time > pingIntervalSleep) { time = cu; return true; } else { return false; } } private volatile long bwtime = System.nanoTime(); private volatile long bwpingInterval = 10000000L; private volatile long bwpingIntervalSleep = 100000000L; private boolean checkBandwidthReportTime(){ long cu = System.nanoTime(); if (cu - bwtime > bwpingIntervalSleep) { bwtime = cu; return true; } else { return false; } } private volatile Consumer rec; private volatile Thread tlock; private ConcurrentLinkedQueue IsendDequeList = new ConcurrentLinkedQueue(); private PriorityBlockingQueue sendDequeList = new PriorityBlockingQueue(); public PriorityBlockingQueue getQueue() { return sendDequeList; } protected void sendPacket(KLALBPacket blk) { blk.markJoinqueuetime(); sendDequeList.add(blk); LockSupport.unpark(tlock); } private void sendIPacket(KLALBPacket blk) { IsendDequeList.add(blk); LockSupport.unpark(tlock); } public void setPacketReceiver(Consumer< KLALBPacket> rec) { this.rec = rec; } /*public void remoeFromSendQueue(KLALBPacket klalbPacket) { sendDequeList.remove(klalbPacket); }*/ public long statLengthBefore(long l) { AtomicLong al = new AtomicLong(); for (Iterator iterator = sendDequeList.iterator(); iterator.hasNext();) { KLALBPacket queue = iterator.next(); if (queue.getPriority() <= l) { al.addAndGet(queue.getLength()); } } return al.get(); } private volatile long predictTime; public void runPredict() { //predictTime = monitor.getOutDelayPredicted(); predictTime=monitor.getOutDelay(); /* * long al=0; al=statLengthBefore(5); long speed=getMonitor().getOutSpeedAvg(); * // spdlmt.setLimitspeed(speed*2+65536); if(speed==0) { if(al>0) { * predictTime=Long.MAX_VALUE; } }else { predictTime+=al*1000000000L/speed; } */ } @Override public int compareTo(KLALBRemoteLine o2) { long t1 = this.predictTime; long t2 = o2.predictTime; if (t1 > t2) { return 1; } else if (t1 < t2) { return -1; } else { return 0; } } public void reconnectImmediately() { if (lthd != null) { LockSupport.unpark(lthd); } } public void dislink() { if (kplink != null) { try { kplink.close(); } catch (IOException e) { } } } private static final TESTPacket testPacket = new TESTPacket(60000); private volatile boolean pressure = false; private SpeedLimiter pressureSpeed = new SpeedLimiter(32768); public void pressureTest() { if (pressure) throw new IllegalStateException("Test already start"); this.pressure = true; pressureSpeed.setLimitspeed(32768); ThreadTool.makeVThreadIfSupport("压力测试控制线程", () -> { while (pressure) { try { Thread.sleep(100); } catch (InterruptedException e) { e.printStackTrace(); } if (getMonitor().getOutSpeed() < pressureSpeed.getLimitspeed() * 2 / 3 || getMonitor().getInDelayMin() * 30 < getMonitor().getInDelay()) { pressure = false; break; } pressureSpeed.setLimitspeed(pressureSpeed.getLimitspeed() * 400 / 399); } }).start(); } public long getPredictedTime() { return predictTime; } private Listranks=new ArrayList(); private Consumer ipv6con; @Override public void noticeRank(int rank) { while(rank>=ranks.size()) { ranks.add(0.5); } for (int i = 0; i < ranks.size(); i++) { if(i==rank) { ranks.set(i, (ranks.get(i)*99.0+1.0D)/100.0); }else { ranks.set(i, ranks.get(i)*99.0/100.0); } } } @Override public double getWeightAtRank(int rank) { while(rank>=ranks.size()) { ranks.add(0.5); } return ranks.get(rank); } public List getRanks() { return ranks; } @Override public boolean isLoopBack() { return false; } @Override public List getNeighborsInfo() { Lisths=new ArrayList<>(); if(peerAddress!=null) { hs.add(new Neighbor(peerAddress, remoteVaddr, monitor)); //System.out.println(peerAddress+" "+addressGroup); } return hs; } @Override public void sendPacket(IPv6Packet pack, Inet6Address inet6Address) throws IOException { pack.lockAll(); try { if(pack.isSomeDisposed()) { return; } if(inet6Address.equals(remoteVaddr.getAddress())||inet6Address.equals(peerAddress.getAddress())) { IPv6OverKLALBPacket kipv6=new IPv6OverKLALBPacket(pack); kipv6.setPriority(pack.getPriority()); kipv6.setDisposeAfterSend(true); pack.putTimePassport("packdInLink"); pack.printPassport(); sendPacketToLink(kipv6); } }finally { pack.unlockAll(); } } private void sendPacketToLink(KLALBPacket packet) { packet.genseq(); //packet.getSendRecord().add(this); sendPacket(packet); } @Override public boolean isCongress(IPv6Packet iPv6Packet) { int size=getQueue().size(); if(size>30) { return true; }else { return !congress.checkTransmit(iPv6Packet.getLength()); } } @Override public String getName() { return monitor.getName(); } @Override public boolean isUp() { return (!isClosed())&&getMonitor().getState()==MonitorData.ONLINE; } @Override public boolean canSend(IPv6Packet iPv6Packet) { return true; } @Override public void setReceiveConsumer(Consumer ipv6con) { this.ipv6con=ipv6con; } }