agkmeans and extention

examples/shed.py

@@ -9,9 +9,16 @@ from multiprocessing import Pool
 from os import path
 from pprint import pprint
 
+<<<<<<< HEAD
 from pydtnsim import Network, RandomTraffic, Node, EpidemicNode, CSVTrace
 from pydtnsim.community import BubbleKCliqueNode, BubbleLouvainNode
 from pydtnsim.community import HCBFKCliqueNode, HCBFLouvainNode
+=======
+from pydtn import Network, RandomTraffic, Node, EpidemicNode, CSVTrace
+from pydtn.community import BubbleKCliqueNode, BubbleLouvainNode
+from pydtn.community import HCBFKCliqueNode, HCBFLouvainNode
+from pydtn.community import  HCBFAGKmeansNode, BubbleAGKmeansNode
+>>>>>>> d960e624440fdfe57ae489eb5130565794d246d0
 
 
 Simulation = namedtuple('Simulation', ['trace', 'node_type', 'seed'])
@@ -32,6 +39,7 @@ def run_simulation(simulation):
         'tick_rate': 5 * 60,  # 5 mins
         'epoch': epoch,
         'k': 3,
+        'agk':3,
         'context': {}
     }
     nodes = {
@@ -70,17 +78,18 @@ def main(args):
     node_types = [
         Node,
         EpidemicNode,
-        BubbleKCliqueNode,
-        HCBFKCliqueNode,
+        BubbleAGKmeansNode,
+        HCBFAGKmeansNode,
         BubbleLouvainNode,
         HCBFLouvainNode,
+        BubbleKCliqueNode,
+        HCBFKCliqueNode,
     ]
 
     for seed in args['seeds']:
         for node_type in node_types:
             sim = Simulation(trace=trace, node_type=node_type, seed=seed)
             simulations.append(sim)
-
     for stats in pool.imap_unordered(run_simulation, simulations):
         log(stats)
 

kmeans/kmeans.py

+import numpy as np
+import pandas as pd
+import networkx as nx
+from collections import defaultdict
+__author__ = 'Fatemeh Zare<faz361@mail.usask.ca>'
+"""
+ Advanced graph-based k-means is an appropriate algorithm for the datasets which have lots of dynamic objects.
+ Instead of passing through space, Advanced graph-based k-means can only pass through certain points. Thus, it can be applied to datasets
+which can be modeled as graphs.
+In Advanced graph-based k-means, initial centroids would be selected based on
+Global Unique Interaction instead of random selection. Wisely selection of initial
+centroids would decrease the probability of falling into the local optimal clustering.
+"""
+
+
+
+def agkmeans(G, k):
+
+    index_list1 = []
+    for i in G.node:
+        index_list1.append(i)
+    df_initial = pd.DataFrame(columns=['node', 'degree', 'centroid'], index=index_list1)
+    df_initial = df_initial.reset_index(drop=True)
+    p = 0
+    for i in G.node:
+        df_initial["node"][p] = i
+        df_initial["degree"][p] = G.degree(i)
+        p = p + 1
+
+    df_initial.sort_values("degree", inplace=True, ascending=False)
+    df_initial = df_initial.reset_index(drop=True)
+
+    k_list = []
+    for d in range(0, k):
+        k_list.append(df_initial['node'][d])
+
+    df2 = pd.DataFrame(columns=['node', 'shoretest_path_centroid', 'centroid'])
+    df2['node'] = df_initial['node']
+
+    def assignment1(G, df, randomlist):
+        df['shoretest_path_centroid'] = float('inf')
+        df['centroid'] = None
+        for j in randomlist:
+            for i in G.node:
+                b = df.loc[df['node'] == i, 'shoretest_path_centroid'].iloc[0]
+
+                try:
+                    n = nx.shortest_path_length(G, i, j)
+                except nx.NetworkXNoPath:
+                    n=1000000000000000000
+                if n < b:
+                    df.loc[df['node'] == i, 'shoretest_path_centroid'] = n
+                    df.loc[df['node'] == i, 'centroid'] = j
+
+        randomlist.sort()
+        return (randomlist)
+
+    def update(G, df2, k_list):
+        df2_sum = df2.groupby(['centroid'])['shoretest_path_centroid'].sum()
+
+        for i in k_list:
+            for j in G.neighbors(i):
+                p = 0
+                sump = 0
+                for b in G.node:
+                    if (df2.loc[df2['node'] == b, 'centroid'].iloc[0] == i):
+                        try:
+                            p = nx.shortest_path_length(G, j, b)
+                        except nx.NetworkXNoPath:
+                            p = float('inf')
+
+                        sump += p
+                if sump < df2_sum[i]:
+                    df2.loc[df2['centroid'] == i, 'centroid'] = j
+
+        for i in df2['node']:
+            b = df2.loc[df2['node'] == i, 'centroid'].iloc[0]
+            try:
+                v = nx.shortest_path_length(G, i, b)
+            except nx.NetworkXNoPath:
+                v = float('inf')
+            df2.loc[df2['node'] == i, 'shoretest_path_centroid'] = v
+#
+        lista = df2['centroid'].unique().tolist()
+        return (lista)
+
+    a=assignment1(G, df2, k_list)
+
+    b=update(G, df2, k_list)
+
+    while (a.sort() != b.sort()):
+        k_list = b
+        a=assignment1(G, df2, k_list)
+        b=update(G, df2, k_list)
+
+    dict1=df2.set_index('node')['centroid'].to_dict()
+    return (dict1)
+

pydtn-develop (2) @ 5623647f

+Subproject commit 5623647f0a969c37d838e7b43e0cdf9a6229c36f

pydtnsim/__init__.py

@@ -9,6 +9,7 @@ A simulation is made up from a Network which contains:
   source, and which node is the destination.
 """
 
+
 __all__ = [
     'Network',
     'Buffer',
@@ -31,7 +32,8 @@ __all__ = [
     'RandomTraffic',
 ]
 __version__ = '0.2'
-__author__ = 'Jarrod Pas <j.pas@usask.ca>'
+__author__ = 'Jarrod Pas <j.pas@usask.ca>, Fatemeh Zare <faz361@mail.usask.ca>'
+
 
 from collections import ChainMap, defaultdict, namedtuple, OrderedDict
 import csv
@@ -39,9 +41,18 @@ import json
 from itertools import count
 from random import Random
 from time import time
-
+from array import array
+from operator import itemgetter
+import operator
+from random import *
+import random as ra
+import numpy as np
+import numpy.random as rand
 import simpy
+import pprint
+import datetime
 
+delayarray=[]
 
 class Network:
     """
@@ -105,9 +116,13 @@ class Network:
         for contact in self.trace:
             if contact.time > self.now:
                 yield self.env.timeout(contact.time - self.now)
-
+            #defining name for each node, node.name  is the actuall name of each node which can used for tracking based on dataset
             node_a = self.nodes[contact.a]
+            node_a.name=contact.a
+
             node_b = self.nodes[contact.b]
+            node_b.name = contact.b
+
 
             if contact.join:
                 node_a.join(node_b)
@@ -144,11 +159,13 @@ class Network:
         """Return detailed statistics for the simulation."""
         packets = [packet.stats for packet in self.packets]
         nodes = [node.stats for node in self.nodes.values()]
+        # hoops = [hoop.stats for packet in self.hoops]
 
         stats = {
             'sim-time': self.env.now,
             'packets': packets,
             'nodes': nodes,
+
         }
 
         stats.update(self._stats)
@@ -167,6 +184,7 @@ class Network:
 
         packets = gather(self.packets)
         nodes = gather(self.nodes.values())
+        # hoops=gather(self.hoops)
 
         stats = {}
         stats['sim-time'] = self.env.now
@@ -176,7 +194,7 @@ class Network:
         stats['delivery-ratio'] = stats['recieved'] / stats['packets']
         stats['broadcasts'] = sum(nodes['broadcasts'])
         stats['delivery-cost'] = stats['broadcasts'] / stats['recieved']
-
+        print(delayarray)
         for stat, values in packets.items():
             if stat.startswith('hops'):
                 stats[stat] = sum(values)
@@ -199,11 +217,16 @@ class Packet:
         self.info = info
         self.recieved = None
         self._stats = defaultdict(int)
+        # self.hoops=None
+        # self.hhop=None
+        # self.energy=None
+        # self.copy=None
 
     @property
     def source(self):
         """Source of packet."""
         return self.network.nodes[self.info.source]
+        # print(self.info.source)
 
     @property
     def destination(self):
@@ -237,16 +260,31 @@ class Packet:
 
     def sent(self, target, reason=None):
         """Send the packet from source to taget."""
+
         self._stats['hops'] += 1
+
         if reason:
             self._stats['hops-%s' % reason] += 1
 
         if target is self.destination:
+
             if self.recieved is None:
                 self.recieved = 0
-                self._stats['delay'] = self.network.now - self.created
+
+            # a sample code for extracting dutycycle delay per packet
+            # same code can be applied for the number of hop per packet
+                self.DCdelay=0
+            self._stats['delay'] = self.network.now - self.created
+            self.DCdelay= self.network.now - self.created
+            self._stats['DCdelay'] += (self.DCdelay/300)
+            delayarray.append(self._stats['DCdelay'])
+
+
+
             self.recieved += 1
 
+
+
     @property
     def stats(self):
         """Return statistcs for a packet."""
@@ -254,6 +292,8 @@ class Packet:
 
         if self.recieved:
             stats['recieved'] = self.recieved
+            stats['DCdelay']=self.DCdelay
+            # stats['energy']=self.energy
 
         stats.update(self._stats)
         return stats
@@ -285,11 +325,20 @@ class Buffer:
         else:
             self.capacity = capacity
         self.store = OrderedDict()
+        self.counter=0
 
     @property
     def full(self):
         """Is the buffer full."""
-        return len(self.store) >= self.capacity
+        # for packet in self.store:
+
+
+        return len(self.store)>= self.capacity
+
+    def counteri(self):
+        self.counter+=1
+        return(self.counter)
+
 
     def add(self, packet):
         """Add a packet to the buffer."""
@@ -298,6 +347,7 @@ class Buffer:
         self.store[packet] = None
         return True
 
+
     def remove(self, packet):
         """Remove packet from the buffer."""
         if packet in self.store:
@@ -325,17 +375,19 @@ class Node:
 
     class SendFailed(Exception):
         """Raised when a send fails."""
-
+    file_flag=False
+    file_date_time=None
     def __init__(self, **options):
         """Create a node."""
         self.network = None
-
+        self.name=None
         self.buffer = Buffer(capacity=options.get('buffer_capacity', None))
         self._neighbours = set()
 
+
         self.options = ChainMap(options, {
             'bandwidth': float('inf'),
-            'tick_rate': 1,
+            'tick_rate': 1
         })
 
         self._stats = defaultdict(int)
@@ -380,6 +432,14 @@ class Node:
         if node in self._neighbours:
             self._neighbours.remove(node)
 
+    #recording the packet transmission in a file
+    def record(self,fname, message):
+        if Node.file_flag==False:
+                Node.file_flag=True
+                Node.file_date_time=str(datetime.datetime.now())
+        file_name=fname+"_"+'.txt'
+        with open(file_name,'a') as file:
+            file.write(message+'\n')
     @property
     def neighbours(self):
         """Return nodes within transmission range."""
@@ -388,31 +448,54 @@ class Node:
     def forward(self, packet):
         """Forward packet directly to their destination if possible."""
         if packet.destination in self.neighbours:
+
             return {packet.destination: 'direct'}
         # forward to nobody
         return {}
 
+
+
+
     def forward_all(self):
         """Try to forward all packets."""
         env = self.network.env
 
         for packet in self.buffer:
+            bw=len(packet)
             if packet.expired:
                 self.packet_expiry(packet)
                 continue
 
+
             forwards = self.forward(packet)
+
             if forwards:
                 delay = len(packet) / self.options['bandwidth']
+
                 yield env.timeout(delay)
+
+
                 self._stats['broadcasts'] += 1
 
+            # counting number of copies for each packet. useful for epidemic
+            if packet._stats['copy'] is None:
+                packet._stats['copy']=1
+            packetcopy=0
             for target, reason in forwards.items():
-                try:
-                    self.send(packet, target, reason)
-                    self.send_success(packet, target)
-                except Node.SendFailed:
-                    self.send_failure(packet, target)
+                packetcopy+=1
+                if(packetcopy>=2):
+                    packet._stats['copy']+=1
+
+            #setting bandwidth limitation
+                if (bw <= self.options['bandwidth']):
+
+                    try:
+                        self.send(packet, target, reason)
+                        self.send_success(packet, target)
+                        bw += len(packet)
+
+                    except Node.SendFailed:
+                        self.send_failure(packet, target)
 
     def send(self, packet, target, reason=None):
         """
@@ -421,12 +504,17 @@ class Node:
         Keyword Arguments:
         reason -- why the packet was sent (optional)
         """
+        #having a track for  each message transmission in a file
         if target in self.neighbours:
+            self.buffer.counteri()
             target.recv(packet, source=target)
             packet.sent(target, reason=reason)
+            message="N["+str(self.name)+'] send to N'+str(target.name)+']'
+            self.record('send_data',message=message)
         else:
             raise Node.SendFailed('nodes are not neighbours')
 
+        # delay = len(packet) / self.options['bandwidth']
     def send_success(self, packet, target):
         """
         Call when a send succeeds.
@@ -444,6 +532,9 @@ class Node:
         """Call when a packet expires."""
         self.buffer.remove(packet)
 
+
+
+
     def recv(self, packet, source=None):
         """
         Recieve a packet.
@@ -455,7 +546,20 @@ class Node:
             return
 
         if not self.buffer.add(packet):
-            raise Node.SendFailed('buffer full')
+            deadlinearray = []
+            for item in self.buffer.store:
+                deadlinearray.append(item.deadline)
+
+            deadlinemin = (min(deadlinearray))
+            if (packet.deadline> deadlinemin):
+
+                for packeti in self.buffer.store:
+                    if packeti.deadline==deadlinemin:
+                        self.buffer.remove(packeti)
+                        self.buffer.add(packet)
+                        break
+
+
 
     @property
     def stats(self):
@@ -478,6 +582,8 @@ class EpidemicNode(Node):
         super().__init__(**options)
         self.__sent = defaultdict(set)
 
+
+
     def forward(self, packet):
         """
         Forward based on the epidemic heuristic.
@@ -488,12 +594,14 @@ class EpidemicNode(Node):
         Epidemic Heuristic:
         - Forward the packet to all neighbours that I have not forwarded
           to so far.
+
         """
+
         forward = {
-            neighbour: 'epidemic'
-            for neighbour in self.neighbours
-            if neighbour not in self.__sent[packet]
-        }
+                neighbour: 'epidemic'
+                for neighbour in self.neighbours
+                if neighbour not in self.__sent[packet]
+                }
 
         return forward
 
@@ -592,12 +700,16 @@ class CSVTrace(Trace):
         if metadata is not None:
             with open(metadata) as metadata_file:
                 nodes = json.load(metadata_file)['nodes']
+
+
+
         else:
             with open(self.path) as csv_file:
                 csv_file = csv.reader(csv_file)
                 next(csv_file)
                 nodes = set()
                 for _, node_a, node_b, _ in csv_file:
+                    # print(node_a)
                     nodes.add(node_a)
                     nodes.add(node_b)
                 nodes = len(nodes)
@@ -641,8 +753,9 @@ class Traffic:
 
     def __init__(self, **options):
         """Create traffic generator."""
-        self.time_to_live = options.get('time_to_live', float('inf'))
-        self.payload = options.get('payload', 0)
+        self.time_to_live = options.get('time_to_live',float('inf'))
+        self.payload = options.get('payload',0)
+
 
     def __iter__(self):
         """Yield no traffic."""
@@ -650,6 +763,7 @@ class Traffic:
 
     def create_packet(self, now, source, destination):
         """Create a packet info for the simulation."""
+
         return PacketInfo(source=source,
                           destination=destination,
                           created=now,
@@ -687,7 +801,10 @@ class RandomManyToManyTraffic(Traffic):
 
     def random_pair(self, random):
         """Return random pair of nodes."""
-        source = destination = random.choice(self.sources)
+        source= random.choice(self.sources)
+        destination = random.choice(self.destinations)
+
+
         while destination == source:
             destination = random.choice(self.destinations)
         return source, destination

pydtnsim/community.py

@@ -14,18 +14,22 @@ __all__ = [
     "Community",
     "KCliqueCommunity",
     "LouvainCommunity",
+    "AGKmeansCommunity",
 
     "CommunityNode",
     "KCliqueNode",
+    "AGKmeansNode",
     "LouvainNode",
     "BubbleNode",
     "BubbleKCliqueNode",
+    "BubbleAGKmeansNode",
     "BubbleLouvainNode",
     "HCBFNode",
     "HCBFKCliqueNode",
+    "HCBFAGKmeansNode",
     "HCBFLouvainNode",
 ]
-__author__ = 'Jarrod Pas <j.pas@usask.ca>'
+__author__ = 'Jarrod Pas <j.pas@usask.ca>, Fatemeh Zare <faz361@mail.usask.ca>'
 
 from collections import defaultdict
 from itertools import product
@@ -34,7 +38,12 @@ from functools import partial
 import networkx as nx
 from community import best_partition as louvain_partition
 
+<<<<<<< HEAD:pydtnsim/community.py
 from pydtnsim import Node
+=======
+from pydtn import Node
+from  kmeans.kmeans import agkmeans
+>>>>>>> d960e624440fdfe57ae489eb5130565794d246d0:pydtn/community.py
 
 
 class Community:
@@ -221,6 +230,36 @@ class KCliqueCommunity(Community):
         """Partition graph by k-clique communities method."""
         return nx.k_clique_communities(self.graph_old, k=self.k)
 
+class AGKmeansCommunity(Community):
+    """AGK-means community detection."""
+
+    def __init__(self, epoch, agk):
+        """Create a k-clique community detector."""
+        super().__init__(epoch)
+        self.agk = agk
+
+    def partition(self):
+        """Partition graph by AGk-means communities method."""
+
+        graph = nx.Graph()
+        nodes = {
+            id(node): node
+            for node in self.graph_old.nodes()
+        }
+        graph.add_nodes_from(nodes)
+        for node_a, node_b, weight in self.graph_old.edges(data='weight'):
+            graph.add_edge(id(node_a), id(node_b), {'weight': weight})
+
+        partitions = agkmeans(graph,k=self.agk)
+
+        communities = defaultdict(set)
+        for node, community in partitions.items():
+            # get actual node from it's id
+            communities[community].add(nodes[node])
+
+        for community in communities.values():
+            yield frozenset(community)
+
 
 class LouvainCommunity(Community):
     """Louviain community detection."""
@@ -250,6 +289,7 @@ class LouvainCommunity(Community):
             yield frozenset(community)
 
 
+
 class CommunityNode(Node):
     """Base node for community based forwarding heuristics."""
 
@@ -354,6 +394,21 @@ class KCliqueNode(CommunityNode):
         options['community'] = shared_community(options['context'], _create)
         super().__init__(**options)
 
+class AGKmeansNode(CommunityNode):
+    """Wrapper to add kclique community detection automatically to a node."""
+
+    def __init__(self, **options):
+        """
+        Create k-clique node.
+
+        Keyword Arguments:
+        epoch -- how often to recalculate communities
+        k -- initial community size
+        """
+        def _create():
+            return AGKmeansCommunity(options['epoch'], options['agk'])
+        options['community'] = shared_community(options['context'], _create)
+        super().__init__(**options)
 
 def _decide(node, others, key):
     """
@@ -407,6 +462,10 @@ class BubbleKCliqueNode(KCliqueNode, BubbleNode):
     """Bubble node with k-clique community detection."""
 
 
+class BubbleAGKmeansNode(AGKmeansNode, BubbleNode):
+    """Bubble node with k-clique community detection."""
+
+
 class BubbleLouvainNode(LouvainNode, BubbleNode):
     """Bubble node with louvain community detection."""
 
@@ -460,5 +519,9 @@ class HCBFKCliqueNode(KCliqueNode, HCBFNode):
     """HCBF node with k-clique community detection."""
 
 
+class HCBFAGKmeansNode(AGKmeansNode, HCBFNode):
+    """HCBF node with k-clique community detection."""
+
+
 class HCBFLouvainNode(LouvainNode, HCBFNode):
     """HCBF node with louvain community detection."""